Merge pull request #3866 from deitch/cache-clean-published

add options to clean only part of the cache

src/cmd/linuxkit/cache.go

@@ -32,6 +32,8 @@ func cache(args []string) {
 	// Please keep cases in alphabetical order
 	case "clean":
 		cacheClean(args[1:])
+	case "rm":
+		cacheRm(args[1:])
 	case "ls":
 		cacheList(args[1:])
 	case "export":

src/cmd/linuxkit/cache/image.go

@@ -1,13 +1,20 @@
 package cache
 
 import (
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ListImages list the named images and their root digests from a layout.Path
-func ListImages(p layout.Path) (map[string]string, error) {
-	ii, err := p.ImageIndex()
+func ListImages(dir string) (map[string]string, error) {
+	p, err := NewProvider(dir)
+	if err != nil {
+		return nil, err
+	}
+	return p.List()
+}
+
+func (p *Provider) List() (map[string]string, error) {
+	ii, err := p.cache.ImageIndex()
 	if err != nil {
 		return nil, err
 	}

src/cmd/linuxkit/cache/remove.go

@@ -0,0 +1,91 @@
+package cache
+
+import (
+	"fmt"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/match"
+	log "github.com/sirupsen/logrus"
+)
+
+// Remove removes all references pointed to by the provided reference, whether it is an image or an index.
+// If it is not found, it is a no-op. This should be viewed as "Ensure this reference is not in the cache",
+// rather than "Remove this reference from the cache".
+func (p *Provider) Remove(name string) error {
+	root, err := p.FindRoot(name)
+	if err != nil {
+		return err
+	}
+	var blobs []v1.Hash
+	// the provided name could be an image or an index, so we need to check both
+	img, err := root.Image()
+	if err == nil {
+		imgBlobs, err := blobsForImage(img)
+		if err != nil {
+			return err
+		}
+		blobs = append(blobs, imgBlobs...)
+		imgDigest, err := img.Digest()
+		if err != nil {
+			return err
+		}
+		blobs = append(blobs, imgDigest)
+	} else {
+		ii, err := root.ImageIndex()
+		if err != nil {
+			return nil
+		}
+		// get blobs for each provided image
+		manifests, err := ii.IndexManifest()
+		if err != nil {
+			return fmt.Errorf("unable to list manifests in index for %s: %v", name, err)
+		}
+		for _, man := range manifests.Manifests {
+			img, err := ii.Image(man.Digest)
+			if err != nil {
+				return fmt.Errorf("unable to get image for digest %s in index for %s: %v", man.Digest, name, err)
+			}
+			imgBlobs, err := blobsForImage(img)
+			if err != nil {
+				return err
+			}
+			blobs = append(blobs, imgBlobs...)
+			blobs = append(blobs, man.Digest)
+		}
+		indexDigest, err := ii.Digest()
+		if err != nil {
+			return err
+		}
+		blobs = append(blobs, indexDigest)
+	}
+	// at this point, blobs contains all of the blobs that need to be removed.
+	for _, blob := range blobs {
+		log.Debugf("removing blob %s", blob)
+		if err := p.cache.RemoveBlob(blob); err != nil {
+			log.Warnf("unable to remove blob %s for %s: %v", blob, name, err)
+		}
+	}
+	return p.cache.RemoveDescriptors(match.Name(name))
+}
+
+func blobsForImage(img v1.Image) ([]v1.Hash, error) {
+	var blobs []v1.Hash
+	layers, err := img.Layers()
+	if err != nil {
+		// if we could not find the layers locally, that is fine;
+		// we are trying to ensure they don't exist in the cache,
+		// and they already don't exist.
+		return nil, nil
+	}
+	for _, layer := range layers {
+		dig, err := layer.Digest()
+		if err != nil {
+			return nil, err
+		}
+		blobs = append(blobs, dig)
+	}
+	if config, err := img.ConfigName(); err == nil {
+		blobs = append(blobs, config)
+	}
+	return blobs, nil
+}

src/cmd/linuxkit/cache/resolvabledescriptor.go

@@ -3,7 +3,7 @@ package cache
 import (
 	"fmt"
 
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 )
@@ -17,6 +17,7 @@ import (
 type ResolvableDescriptor interface {
 	Image() (v1.Image, error)
 	ImageIndex() (v1.ImageIndex, error)
+	Digest() (v1.Hash, error)
 }
 type layoutImage struct {
 	img v1.Image
@@ -28,6 +29,9 @@ func (l layoutImage) Image() (v1.Image, error) {
 func (l layoutImage) ImageIndex() (v1.ImageIndex, error) {
 	return nil, fmt.Errorf("not an ImageIndex")
 }
+func (l layoutImage) Digest() (v1.Hash, error) {
+	return l.img.Digest()
+}
 
 type layoutIndex struct {
 	idx v1.ImageIndex
@@ -39,6 +43,9 @@ func (l layoutIndex) Image() (v1.Image, error) {
 func (l layoutIndex) ImageIndex() (v1.ImageIndex, error) {
 	return l.idx, nil
 }
+func (l layoutIndex) Digest() (v1.Hash, error) {
+	return l.idx.Digest()
+}
 
 // FindRoot find the root ResolvableDescriptor, representing an Image or Index, for
 // a given imageName.

src/cmd/linuxkit/cache/write.go

@@ -49,12 +49,12 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin
 	if !alwaysPull {
 		imgSrc, err := p.ValidateImage(ref, architecture)
 		if err == nil && imgSrc != nil {
-			log.Printf("Image %s found in local cache, not pulling", image)
+			log.Printf("Image %s arch %s found in local cache, not pulling", image, architecture)
 			return imgSrc, nil
 		}
 		// there was an error, so try to pull
 	}
-	log.Printf("Image %s not found in local cache, pulling", image)
+	log.Printf("Image %s arch %s not found in local cache, pulling", image, architecture)
 	remoteRef, err := name.ParseReference(pullImageName)
 	if err != nil {
 		return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err)
@@ -78,18 +78,21 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin
 		if err != nil {
 			return ImageSource{}, fmt.Errorf("unable to get IndexManifest: %v", err)
 		}
-		_, err = p.IndexWrite(ref, im.Manifests...)
-		if err == nil {
-			for _, m := range im.Manifests {
-				if m.MediaType.IsImage() && (m.Platform == nil || m.Platform.Architecture == architecture) {
-					img, err := ii.Image(m.Digest)
-					if err != nil {
-						return ImageSource{}, fmt.Errorf("unable to get image: %v", err)
-					}
-					err = p.cache.WriteImage(img)
-					if err != nil {
-						return ImageSource{}, fmt.Errorf("unable to write image: %v", err)
-					}
+		// write the index blob and the descriptor
+		if err := p.cache.WriteBlob(desc.Digest, io.NopCloser(bytes.NewReader(desc.Manifest))); err != nil {
+			return ImageSource{}, fmt.Errorf("unable to write index content to cache: %v", err)
+		}
+		if _, err := p.DescriptorWrite(ref, desc.Descriptor); err != nil {
+			return ImageSource{}, fmt.Errorf("unable to write index descriptor to cache: %v", err)
+		}
+		for _, m := range im.Manifests {
+			if m.MediaType.IsImage() && (m.Platform == nil || m.Platform.Architecture == architecture) {
+				img, err := ii.Image(m.Digest)
+				if err != nil {
+					return ImageSource{}, fmt.Errorf("unable to get image: %v", err)
+				}
+				if err := p.cache.WriteImage(img); err != nil {
+					return ImageSource{}, fmt.Errorf("unable to write image: %v", err)
 				}
 			}
 		}
@@ -355,9 +358,39 @@ func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) (lkt
 }
 
 func (p *Provider) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
-	if _, err := p.findImage(ref.String(), architecture); err != nil {
+	img, err := p.findImage(ref.String(), architecture)
+	if err != nil {
 		return false, err
 	}
+	// findImage only checks if we had the pointer to it; it does not check if it is complete.
+	// We need to do that next.
+
+	// check that all of the layers exist
+	layers, err := img.Layers()
+	if err != nil {
+		return false, fmt.Errorf("layers not found: %v", err)
+	}
+	for _, layer := range layers {
+		dig, err := layer.Digest()
+		if err != nil {
+			return false, fmt.Errorf("unable to get digest of layer: %v", err)
+		}
+		var rc io.ReadCloser
+		if rc, err = p.cache.Blob(dig); err != nil {
+			return false, fmt.Errorf("layer %s not found: %v", dig, err)
+		}
+		rc.Close()
+	}
+	// check that the config exists
+	config, err := img.ConfigName()
+	if err != nil {
+		return false, fmt.Errorf("unable to get config: %v", err)
+	}
+	var rc io.ReadCloser
+	if rc, err = p.cache.Blob(config); err != nil {
+		return false, fmt.Errorf("config %s not found: %v", config, err)
+	}
+	rc.Close()
 	return true, nil
 }
 

src/cmd/linuxkit/cache_clean.go

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"os"
 
+	namepkg "github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	cachepkg "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -13,13 +16,62 @@ func cacheClean(args []string) {
 
 	cacheDir := flagOverEnvVarOverDefaultString{def: defaultLinuxkitCache(), envVar: envVarCacheDir}
 	flags.Var(&cacheDir, "cache", fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))
+	publishedOnly := flags.Bool("published-only", false, "Only clean images that linuxkit can confirm at the time of running have been published to the registry")
 
 	if err := flags.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
 	}
 
-	if err := os.RemoveAll(cacheDir.String()); err != nil {
-		log.Fatalf("Unable to clean cache %s: %v", cacheDir, err)
+	// did we limit to published only?
+	if !*publishedOnly {
+		if err := os.RemoveAll(cacheDir.String()); err != nil {
+			log.Fatalf("Unable to clean cache %s: %v", cacheDir, err)
+		}
+		log.Infof("Cache emptied: %s", cacheDir)
+		return
+	}
+
+	// list all of the images and content in the cache
+	p, err := cachepkg.NewProvider(cacheDir.String())
+	if err != nil {
+		log.Fatalf("unable to read a local cache: %v", err)
+	}
+	images, err := p.List()
+
+	if err != nil {
+		log.Fatalf("error reading image names: %v", err)
+	}
+	removeImagesFromCache(images, p, *publishedOnly)
+}
+
+// removeImagesFromCache removes images from the cache.
+func removeImagesFromCache(images map[string]string, p *cachepkg.Provider, publishedOnly bool) {
+	// check each image in the registry. If it exists, remove it here.
+	for name, hash := range images {
+		if publishedOnly {
+			ref, err := namepkg.ParseReference(name)
+			if err != nil {
+				continue
+			}
+			desc, err := remote.Get(ref)
+			if err != nil {
+				log.Debugf("image %s not found in remote registry or error, leaving in cache: %v", name, err)
+				fmt.Fprintf(os.Stderr, "image %s not found in remote registry, leaving in cache", name)
+				continue
+			}
+			if desc == nil {
+				fmt.Fprintf(os.Stderr, "image %s not found in remote registry, leaving in cache", name)
+				continue
+			}
+			if desc.Digest.String() != hash {
+				fmt.Fprintf(os.Stderr, "image %s has mismatched hashes, cache %s vs remote registry %s, leaving in cache", name, hash, desc.Digest.String())
+				continue
+			}
+		}
+		// we have a match, remove it
+		fmt.Fprintf(os.Stderr, "removing image %s from cache", name)
+		if err := p.Remove(name); err != nil {
+			log.Warnf("Unable to remove image %s: %v", name, err)
+		}
 	}
-	log.Infof("Cache cleaned: %s", cacheDir)
 }

src/cmd/linuxkit/cache_ls.go

@@ -19,11 +19,7 @@ func cacheList(args []string) {
 	}
 
 	// list all of the images and content in the cache
-	p, err := cachepkg.Get(cacheDir.String())
-	if err != nil {
-		log.Fatalf("unable to read a local cache: %v", err)
-	}
-	images, err := cachepkg.ListImages(p)
+	images, err := cachepkg.ListImages(cacheDir.String())
 	if err != nil {
 		log.Fatalf("error reading image names: %v", err)
 	}

src/cmd/linuxkit/cache_rm.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	cachepkg "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
+	log "github.com/sirupsen/logrus"
+)
+
+func cacheRm(args []string) {
+	flags := flag.NewFlagSet("rm", flag.ExitOnError)
+
+	cacheDir := flagOverEnvVarOverDefaultString{def: defaultLinuxkitCache(), envVar: envVarCacheDir}
+	flags.Var(&cacheDir, "cache", fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))
+	publishedOnly := flags.Bool("published-only", false, "Only remove the specified images if linuxkit can confirm at the time of running have been published to the registry")
+
+	if err := flags.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+
+	if flags.NArg() == 0 {
+		log.Fatal("Please specify at least one image to remove")
+	}
+
+	imageNames := flags.Args()
+
+	// did we limit to published only?
+
+	// list all of the images and content in the cache
+	p, err := cachepkg.NewProvider(cacheDir.String())
+	if err != nil {
+		log.Fatalf("unable to read a local cache: %v", err)
+	}
+	images := map[string]string{}
+	for _, imageName := range imageNames {
+		desc, err := p.FindRoot(imageName)
+		if err != nil {
+			log.Fatalf("error reading image %s: %v", imageName, err)
+		}
+		dig, err := desc.Digest()
+		if err != nil {
+			log.Fatalf("error reading digest for image %s: %v", imageName, err)
+		}
+		images[imageName] = dig.String()
+	}
+	removeImagesFromCache(images, p, *publishedOnly)
+}