From 911a0de14caf3eca0840d97cb9b079bc00b0a89b Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Wed, 28 Apr 2021 07:55:48 +0300 Subject: [PATCH] handle image already in cache Signed-off-by: Avi Deitcher --- src/cmd/linuxkit/cache/pull.go | 2 +- src/cmd/linuxkit/cache/write.go | 16 ++++++++++++++-- src/cmd/linuxkit/main.go | 6 ++++++ src/cmd/linuxkit/moby/images.go | 8 +------- src/cmd/linuxkit/pkglib/build.go | 16 ++++++++++------ src/cmd/linuxkit/pkglib/build_test.go | 2 +- src/cmd/linuxkit/spec/cache.go | 2 +- 7 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/cmd/linuxkit/cache/pull.go b/src/cmd/linuxkit/cache/pull.go index c1ad2a566..ef3618367 100644 --- a/src/cmd/linuxkit/cache/pull.go +++ b/src/cmd/linuxkit/cache/pull.go @@ -11,7 +11,7 @@ import ( ) // ValidateImage given a reference, validate that it is complete. If not, pull down missing -// components as necessary. +// components as necessary. It also calculates the hash of each component. func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lktspec.ImageSource, error) { var ( imageIndex v1.ImageIndex diff --git a/src/cmd/linuxkit/cache/write.go b/src/cmd/linuxkit/cache/write.go index 9ddc3df2e..f90534e31 100644 --- a/src/cmd/linuxkit/cache/write.go +++ b/src/cmd/linuxkit/cache/write.go @@ -28,8 +28,9 @@ const ( ) // ImagePull takes an image name and pulls it down, writing it locally. It should be -// efficient and only write missing blobs, based on their content hash. -func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) { +// efficient and only write missing blobs, based on their content hash. If the ref already +// exists in the cache, it will niot pull anything, unless alwaysPull is set to true. +func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) { image := ref.String() pullImageName := image remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)} @@ -37,6 +38,17 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin pullImageName = trustedRef } log.Debugf("ImagePull to cache %s trusted reference %s", image, pullImageName) + + // unless alwaysPull is set to true, check locally first + if !alwaysPull { + imgSrc, err := p.ValidateImage(ref, architecture) + if err == nil && imgSrc != nil { + log.Printf("Image %s found in local cache, not pulling", image) + return imgSrc, nil + } + // there was an error, so try to pull + } + log.Printf("Image %s not found in local cache, pulling", image) remoteRef, err := name.ParseReference(pullImageName) if err != nil { return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err) diff --git a/src/cmd/linuxkit/main.go b/src/cmd/linuxkit/main.go index 3b90a199b..8f8385c80 100644 --- a/src/cmd/linuxkit/main.go +++ b/src/cmd/linuxkit/main.go @@ -4,9 +4,11 @@ import ( "flag" "fmt" "io/ioutil" + stdlog "log" "os" "path/filepath" + ggcrlog "github.com/google/go-containerregistry/pkg/logs" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" log "github.com/sirupsen/logrus" @@ -104,7 +106,11 @@ func main() { // Switch back to the standard formatter log.SetFormatter(defaultLogFormatter) log.SetLevel(log.DebugLevel) + // set go-containerregistry logging as well + ggcrlog.Warn = stdlog.New(log.StandardLogger().WriterLevel(log.WarnLevel), "", 0) + ggcrlog.Debug = stdlog.New(log.StandardLogger().WriterLevel(log.DebugLevel), "", 0) } + ggcrlog.Progress = stdlog.New(log.StandardLogger().WriterLevel(log.InfoLevel), "", 0) args := flag.Args() if len(args) < 1 { diff --git a/src/cmd/linuxkit/moby/images.go b/src/cmd/linuxkit/moby/images.go index 93f7979df..76b3ef61e 100644 --- a/src/cmd/linuxkit/moby/images.go +++ b/src/cmd/linuxkit/moby/images.go @@ -36,15 +36,9 @@ func imagePull(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCach } // if we made it here, we either did not have the image, or it was incomplete - return imageLayoutWrite(cacheDir, ref, architecture) -} - -// imageLayoutWrite takes an image name and pulls it down, writing it locally -func imageLayoutWrite(cacheDir string, ref *reference.Spec, architecture string) (lktspec.ImageSource, error) { - image := ref.String() c, err := cache.NewProvider(cacheDir) if err != nil { return nil, err } - return c.ImagePull(ref, image, architecture) + return c.ImagePull(ref, ref.String(), architecture, alwaysPull) } diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index 2681b5402..51c65d076 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -226,15 +226,19 @@ func (p Pkg) Build(bos ...BuildOpt) error { return fmt.Errorf("buildkit not supported, check docker version: %v", err) } + skipBuild := bo.skipBuild if !bo.force { - if _, err := c.ImagePull(&ref, "", arch); err == nil { - fmt.Fprintf(writer, "image already found %s", ref) - return nil + fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", ref) + if _, err := c.ImagePull(&ref, "", arch, false); err == nil { + fmt.Fprintf(writer, "%s found or pulled\n", ref) + skipBuild = true + } else { + fmt.Fprintf(writer, "%s not found\n", ref) } - fmt.Fprintln(writer, "No image pulled, continuing with build") } - if bo.image && !bo.skipBuild { + if bo.image && !skipBuild { + fmt.Fprintf(writer, "building %s\n", ref) var ( args []string descs []v1.Descriptor @@ -375,7 +379,7 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg if err != nil { return nil, fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err) } - if _, err := c.ImagePull(&ref, "", arch); err == nil { + if _, err := c.ImagePull(&ref, "", arch, false); err == nil { fmt.Fprintf(writer, "image already found %s for arch %s", ref, arch) desc, err := c.FindDescriptor(ref.String()) if err != nil { diff --git a/src/cmd/linuxkit/pkglib/build_test.go b/src/cmd/linuxkit/pkglib/build_test.go index bd6d358af..77b90a54c 100644 --- a/src/cmd/linuxkit/pkglib/build_test.go +++ b/src/cmd/linuxkit/pkglib/build_test.go @@ -101,7 +101,7 @@ type cacheMocker struct { hashes map[string][]byte } -func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) { +func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) { if !c.enableImagePull { return nil, errors.New("ImagePull disabled") } diff --git a/src/cmd/linuxkit/spec/cache.go b/src/cmd/linuxkit/spec/cache.go index ed768d6a1..692f58acb 100644 --- a/src/cmd/linuxkit/spec/cache.go +++ b/src/cmd/linuxkit/spec/cache.go @@ -10,7 +10,7 @@ import ( // CacheProvider interface for a provide of a cache. type CacheProvider interface { FindDescriptor(name string) (*v1.Descriptor, error) - ImagePull(ref *reference.Spec, trustedRef, architecture string) (ImageSource, error) + ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (ImageSource, error) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error) ImageLoad(ref *reference.Spec, architecture string, r io.Reader) (ImageSource, error) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)