Merge pull request #3638 from deitch/use-cache

handle image already in cache

src/cmd/linuxkit/cache/pull.go

@@ -11,7 +11,7 @@ import (
 )
 
 // ValidateImage given a reference, validate that it is complete. If not, pull down missing
-// components as necessary.
+// components as necessary. It also calculates the hash of each component.
 func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lktspec.ImageSource, error) {
 	var (
 		imageIndex v1.ImageIndex

src/cmd/linuxkit/cache/write.go

@@ -28,8 +28,9 @@ const (
 )
 
 // ImagePull takes an image name and pulls it down, writing it locally. It should be
-// efficient and only write missing blobs, based on their content hash.
-func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) {
+// efficient and only write missing blobs, based on their content hash. If the ref already
+// exists in the cache, it will niot pull anything, unless alwaysPull is set to true.
+func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
 	image := ref.String()
 	pullImageName := image
 	remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
@@ -37,6 +38,17 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin
 		pullImageName = trustedRef
 	}
 	log.Debugf("ImagePull to cache %s trusted reference %s", image, pullImageName)
+
+	// unless alwaysPull is set to true, check locally first
+	if !alwaysPull {
+		imgSrc, err := p.ValidateImage(ref, architecture)
+		if err == nil && imgSrc != nil {
+			log.Printf("Image %s found in local cache, not pulling", image)
+			return imgSrc, nil
+		}
+		// there was an error, so try to pull
+	}
+	log.Printf("Image %s not found in local cache, pulling", image)
 	remoteRef, err := name.ParseReference(pullImageName)
 	if err != nil {
 		return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err)

src/cmd/linuxkit/main.go

@@ -4,9 +4,11 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	stdlog "log"
 	"os"
 	"path/filepath"
 
+	ggcrlog "github.com/google/go-containerregistry/pkg/logs"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
 
 	log "github.com/sirupsen/logrus"
@@ -104,7 +106,11 @@ func main() {
 		// Switch back to the standard formatter
 		log.SetFormatter(defaultLogFormatter)
 		log.SetLevel(log.DebugLevel)
+		// set go-containerregistry logging as well
+		ggcrlog.Warn = stdlog.New(log.StandardLogger().WriterLevel(log.WarnLevel), "", 0)
+		ggcrlog.Debug = stdlog.New(log.StandardLogger().WriterLevel(log.DebugLevel), "", 0)
 	}
+	ggcrlog.Progress = stdlog.New(log.StandardLogger().WriterLevel(log.InfoLevel), "", 0)
 
 	args := flag.Args()
 	if len(args) < 1 {

src/cmd/linuxkit/moby/images.go

@@ -36,15 +36,9 @@ func imagePull(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCach
 	}
 
 	// if we made it here, we either did not have the image, or it was incomplete
-	return imageLayoutWrite(cacheDir, ref, architecture)
-}
-
-// imageLayoutWrite takes an image name and pulls it down, writing it locally
-func imageLayoutWrite(cacheDir string, ref *reference.Spec, architecture string) (lktspec.ImageSource, error) {
-	image := ref.String()
 	c, err := cache.NewProvider(cacheDir)
 	if err != nil {
 		return nil, err
 	}
-	return c.ImagePull(ref, image, architecture)
+	return c.ImagePull(ref, ref.String(), architecture, alwaysPull)
 }

src/cmd/linuxkit/pkglib/build.go

@@ -226,15 +226,19 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 		return fmt.Errorf("buildkit not supported, check docker version: %v", err)
 	}
 
+	skipBuild := bo.skipBuild
 	if !bo.force {
-		if _, err := c.ImagePull(&ref, "", arch); err == nil {
-			fmt.Fprintf(writer, "image already found %s", ref)
-			return nil
+		fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", ref)
+		if _, err := c.ImagePull(&ref, "", arch, false); err == nil {
+			fmt.Fprintf(writer, "%s found or pulled\n", ref)
+			skipBuild = true
+		} else {
+			fmt.Fprintf(writer, "%s not found\n", ref)
 		}
-		fmt.Fprintln(writer, "No image pulled, continuing with build")
 	}
 
-	if bo.image && !bo.skipBuild {
+	if bo.image && !skipBuild {
+		fmt.Fprintf(writer, "building %s\n", ref)
 		var (
 			args  []string
 			descs []v1.Descriptor
@@ -375,7 +379,7 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg
 		if err != nil {
 			return nil, fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err)
 		}
-		if _, err := c.ImagePull(&ref, "", arch); err == nil {
+		if _, err := c.ImagePull(&ref, "", arch, false); err == nil {
 			fmt.Fprintf(writer, "image already found %s for arch %s", ref, arch)
 			desc, err := c.FindDescriptor(ref.String())
 			if err != nil {

src/cmd/linuxkit/pkglib/build_test.go

@@ -101,7 +101,7 @@ type cacheMocker struct {
 	hashes                 map[string][]byte
 }
 
-func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) {
+func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
 	if !c.enableImagePull {
 		return nil, errors.New("ImagePull disabled")
 	}

src/cmd/linuxkit/spec/cache.go

@@ -10,7 +10,7 @@ import (
 // CacheProvider interface for a provide of a cache.
 type CacheProvider interface {
 	FindDescriptor(name string) (*v1.Descriptor, error)
-	ImagePull(ref *reference.Spec, trustedRef, architecture string) (ImageSource, error)
+	ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (ImageSource, error)
 	IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)
 	ImageLoad(ref *reference.Spec, architecture string, r io.Reader) (ImageSource, error)
 	DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)