handle image already in cache

Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
Avi Deitcher 2021-04-28 07:55:48 +03:00
parent b13b41b213
commit 911a0de14c
7 changed files with 34 additions and 18 deletions

View File

@ -11,7 +11,7 @@ import (
) )
// ValidateImage given a reference, validate that it is complete. If not, pull down missing // ValidateImage given a reference, validate that it is complete. If not, pull down missing
// components as necessary. // components as necessary. It also calculates the hash of each component.
func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lktspec.ImageSource, error) { func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lktspec.ImageSource, error) {
var ( var (
imageIndex v1.ImageIndex imageIndex v1.ImageIndex

View File

@ -28,8 +28,9 @@ const (
) )
// ImagePull takes an image name and pulls it down, writing it locally. It should be // ImagePull takes an image name and pulls it down, writing it locally. It should be
// efficient and only write missing blobs, based on their content hash. // efficient and only write missing blobs, based on their content hash. If the ref already
func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) { // exists in the cache, it will niot pull anything, unless alwaysPull is set to true.
func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
image := ref.String() image := ref.String()
pullImageName := image pullImageName := image
remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)} remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
@ -37,6 +38,17 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin
pullImageName = trustedRef pullImageName = trustedRef
} }
log.Debugf("ImagePull to cache %s trusted reference %s", image, pullImageName) log.Debugf("ImagePull to cache %s trusted reference %s", image, pullImageName)
// unless alwaysPull is set to true, check locally first
if !alwaysPull {
imgSrc, err := p.ValidateImage(ref, architecture)
if err == nil && imgSrc != nil {
log.Printf("Image %s found in local cache, not pulling", image)
return imgSrc, nil
}
// there was an error, so try to pull
}
log.Printf("Image %s not found in local cache, pulling", image)
remoteRef, err := name.ParseReference(pullImageName) remoteRef, err := name.ParseReference(pullImageName)
if err != nil { if err != nil {
return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err) return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err)

View File

@ -4,9 +4,11 @@ import (
"flag" "flag"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
stdlog "log"
"os" "os"
"path/filepath" "path/filepath"
ggcrlog "github.com/google/go-containerregistry/pkg/logs"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -104,7 +106,11 @@ func main() {
// Switch back to the standard formatter // Switch back to the standard formatter
log.SetFormatter(defaultLogFormatter) log.SetFormatter(defaultLogFormatter)
log.SetLevel(log.DebugLevel) log.SetLevel(log.DebugLevel)
// set go-containerregistry logging as well
ggcrlog.Warn = stdlog.New(log.StandardLogger().WriterLevel(log.WarnLevel), "", 0)
ggcrlog.Debug = stdlog.New(log.StandardLogger().WriterLevel(log.DebugLevel), "", 0)
} }
ggcrlog.Progress = stdlog.New(log.StandardLogger().WriterLevel(log.InfoLevel), "", 0)
args := flag.Args() args := flag.Args()
if len(args) < 1 { if len(args) < 1 {

View File

@ -36,15 +36,9 @@ func imagePull(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCach
} }
// if we made it here, we either did not have the image, or it was incomplete // if we made it here, we either did not have the image, or it was incomplete
return imageLayoutWrite(cacheDir, ref, architecture)
}
// imageLayoutWrite takes an image name and pulls it down, writing it locally
func imageLayoutWrite(cacheDir string, ref *reference.Spec, architecture string) (lktspec.ImageSource, error) {
image := ref.String()
c, err := cache.NewProvider(cacheDir) c, err := cache.NewProvider(cacheDir)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return c.ImagePull(ref, image, architecture) return c.ImagePull(ref, ref.String(), architecture, alwaysPull)
} }

View File

@ -226,15 +226,19 @@ func (p Pkg) Build(bos ...BuildOpt) error {
return fmt.Errorf("buildkit not supported, check docker version: %v", err) return fmt.Errorf("buildkit not supported, check docker version: %v", err)
} }
skipBuild := bo.skipBuild
if !bo.force { if !bo.force {
if _, err := c.ImagePull(&ref, "", arch); err == nil { fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", ref)
fmt.Fprintf(writer, "image already found %s", ref) if _, err := c.ImagePull(&ref, "", arch, false); err == nil {
return nil fmt.Fprintf(writer, "%s found or pulled\n", ref)
skipBuild = true
} else {
fmt.Fprintf(writer, "%s not found\n", ref)
} }
fmt.Fprintln(writer, "No image pulled, continuing with build")
} }
if bo.image && !bo.skipBuild { if bo.image && !skipBuild {
fmt.Fprintf(writer, "building %s\n", ref)
var ( var (
args []string args []string
descs []v1.Descriptor descs []v1.Descriptor
@ -375,7 +379,7 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg
if err != nil { if err != nil {
return nil, fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err) return nil, fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err)
} }
if _, err := c.ImagePull(&ref, "", arch); err == nil { if _, err := c.ImagePull(&ref, "", arch, false); err == nil {
fmt.Fprintf(writer, "image already found %s for arch %s", ref, arch) fmt.Fprintf(writer, "image already found %s for arch %s", ref, arch)
desc, err := c.FindDescriptor(ref.String()) desc, err := c.FindDescriptor(ref.String())
if err != nil { if err != nil {

View File

@ -101,7 +101,7 @@ type cacheMocker struct {
hashes map[string][]byte hashes map[string][]byte
} }
func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string) (lktspec.ImageSource, error) { func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
if !c.enableImagePull { if !c.enableImagePull {
return nil, errors.New("ImagePull disabled") return nil, errors.New("ImagePull disabled")
} }

View File

@ -10,7 +10,7 @@ import (
// CacheProvider interface for a provide of a cache. // CacheProvider interface for a provide of a cache.
type CacheProvider interface { type CacheProvider interface {
FindDescriptor(name string) (*v1.Descriptor, error) FindDescriptor(name string) (*v1.Descriptor, error)
ImagePull(ref *reference.Spec, trustedRef, architecture string) (ImageSource, error) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (ImageSource, error)
IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)
ImageLoad(ref *reference.Spec, architecture string, r io.Reader) (ImageSource, error) ImageLoad(ref *reference.Spec, architecture string, r io.Reader) (ImageSource, error)
DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)