From fe46d9bf0589f9ce67abca656cc0bc5c72d6c398 Mon Sep 17 00:00:00 2001 From: Petr Fedchenkov Date: Thu, 28 Jul 2022 12:32:20 +0300 Subject: [PATCH] Do not pull another arch We pull all arches for the image which is suboptimal in terms of storage consumption. Let's pull only required platforms. Signed-off-by: Petr Fedchenkov --- src/cmd/linuxkit/cache/pull.go | 15 +++++++++------ src/cmd/linuxkit/cache/push.go | 6 ++++++ src/cmd/linuxkit/cache/write.go | 31 +++++++++++++++++++++++++++---- src/cmd/linuxkit/pkglib/build.go | 17 ++++++++++------- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/cmd/linuxkit/cache/pull.go b/src/cmd/linuxkit/cache/pull.go index 9af136623..72ea27e4d 100644 --- a/src/cmd/linuxkit/cache/pull.go +++ b/src/cmd/linuxkit/cache/pull.go @@ -54,12 +54,15 @@ func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lkts if err != nil { return ImageSource{}, fmt.Errorf("could not get index manifest: %v", err) } - // we found a local index, just make sure it is up to date and, if not, download it - if err := validate.Index(imageIndex); err != nil { - return ImageSource{}, errors.New("invalid index") - } for _, m := range im.Manifests { - if m.Platform != nil && m.Platform.Architecture == architecture && m.Platform.OS == "linux" { + if m.Platform != nil && m.Platform.Architecture == architecture && m.Platform.OS == linux { + img, err := imageIndex.Image(m.Digest) + if err != nil { + return ImageSource{}, fmt.Errorf("unable to get image: %v", err) + } + if err := validate.Image(img); err != nil { + return ImageSource{}, fmt.Errorf("invalid image: %s", err) + } return p.NewSource( ref, architecture, @@ -71,7 +74,7 @@ func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lkts case image != nil: // we found a local image, make sure it is up to date, and that it matches our platform if err := validate.Image(image); err != nil { - return ImageSource{}, errors.New("invalid image") + return ImageSource{}, fmt.Errorf("invalid image, %s", err) } return p.NewSource( ref, diff --git a/src/cmd/linuxkit/cache/push.go b/src/cmd/linuxkit/cache/push.go index c00c863ea..12870f70b 100644 --- a/src/cmd/linuxkit/cache/push.go +++ b/src/cmd/linuxkit/cache/push.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" namepkg "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry" imagespec "github.com/opencontainers/image-spec/specs-go/v1" log "github.com/sirupsen/logrus" @@ -98,6 +99,11 @@ func (p *Provider) Push(name string) error { return fmt.Errorf("could not find or create arch-specific image for %s: %v", archTag, err) } } + if err := validate.Image(img); err != nil { + // skip arch we did not build/pull locally + log.Debugf("could not validate arch-specific image for %s: %v", archTag, err) + continue + } log.Debugf("pushing image %s", tag) if err := remote.Tag(tag, img, options...); err != nil { return fmt.Errorf("error creating tag %s: %v", archTag, err) diff --git a/src/cmd/linuxkit/cache/write.go b/src/cmd/linuxkit/cache/write.go index 3295fdb43..010483bb9 100644 --- a/src/cmd/linuxkit/cache/write.go +++ b/src/cmd/linuxkit/cache/write.go @@ -28,9 +28,14 @@ const ( linux = "linux" ) -// ImagePull takes an image name and pulls it down, writing it locally. It should be -// efficient and only write missing blobs, based on their content hash. If the ref already -// exists in the cache, it will niot pull anything, unless alwaysPull is set to true. +// ImagePull takes an image name and ensures that the image manifest or index to which it refers +// exists in local cache and, if not, pulls it from the registry and writes it locally. It should be +// efficient and only write missing blobs, based on their content hash. +// It will only pull the actual blobs, config and manifest for the requested architectures, even if ref +// points to an index with multiple architectures. If the ref and all of the content for the requested +// architectures already exist in the cache, it will not pull anything, unless alwaysPull is set to true. +// If you call it multiple times, even with different architectures, the ref will continue to point to the same index. +// Only the underlying content will be added. func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) { image := ref.String() pullImageName := image @@ -69,7 +74,25 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin ii, err := desc.ImageIndex() if err == nil { log.Debugf("ImageWrite retrieved %s is index, saving", pullImageName) - err = p.cache.ReplaceIndex(ii, match.Name(image), layout.WithAnnotations(annotations)) + im, err := ii.IndexManifest() + if err != nil { + return ImageSource{}, fmt.Errorf("unable to get IndexManifest: %v", err) + } + _, err = p.IndexWrite(ref, im.Manifests...) + if err == nil { + for _, m := range im.Manifests { + if m.MediaType.IsImage() && (m.Platform == nil || m.Platform.Architecture == architecture) { + img, err := ii.Image(m.Digest) + if err != nil { + return ImageSource{}, fmt.Errorf("unable to get image: %v", err) + } + err = p.cache.WriteImage(img) + if err != nil { + return ImageSource{}, fmt.Errorf("unable to write image: %v", err) + } + } + } + } } else { var im v1.Image // try an image diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index 1ba60502a..4a14eac16 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -231,15 +231,18 @@ func (p Pkg) Build(bos ...BuildOpt) error { skipBuild := bo.skipBuild if !bo.force { + notFound := false fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", ref) - if _, err := c.ImagePull(&ref, "", arch, false); err == nil { - fmt.Fprintf(writer, "%s found or pulled\n", ref) - skipBuild = true - } else { - fmt.Fprintf(writer, "%s not found\n", ref) + for _, platform := range bo.platforms { + if _, err := c.ImagePull(&ref, "", platform.Architecture, false); err == nil { + fmt.Fprintf(writer, "%s found or pulled\n", ref) + skipBuild = true + } else { + fmt.Fprintf(writer, "%s not found: %s\n", ref, err) + notFound = true + } } if bo.targetDocker { - notFound := false for _, platform := range bo.platforms { archRef, err := reference.Parse(fmt.Sprintf("%s-%s", p.FullTag(), platform.Architecture)) if err != nil { @@ -250,7 +253,7 @@ func (p Pkg) Build(bos ...BuildOpt) error { fmt.Fprintf(writer, "%s found or pulled\n", archRef) skipBuild = true } else { - fmt.Fprintf(writer, "%s not found\n", archRef) + fmt.Fprintf(writer, "%s not found: %s\n", archRef, err) notFound = true } }