provide mirror support (#4148)

Signed-off-by: Avi Deitcher <avi@deitcher.net>

src/cmd/linuxkit/cache/pull.go

@@ -12,6 +12,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/validate"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -175,7 +176,7 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
 	// before we even try to push, let us see if it exists remotely
 	remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
 
-	desc, err := remote.Get(ref, remoteOptions...)
+	desc, err := registry.GetRemote().Get(ref, remoteOptions...)
 	if err != nil {
 		return fmt.Errorf("error getting manifest for trusted image %s: %v", name, err)
 	}

src/cmd/linuxkit/cache/push.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/validate"
 
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	log "github.com/sirupsen/logrus"
 )
@@ -39,7 +40,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
 
 	// check if it already exists, unless override is explicit
 	if !override {
-		if _, err := remote.Get(ref, options...); err == nil {
+		if _, err := registry.GetRemote().Get(ref, options...); err == nil {
 			log.Infof("image %s already exists in the registry, skipping", remoteName)
 			return nil
 		}
@@ -64,7 +65,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
 		if err != nil {
 			return fmt.Errorf("could not get digest for local image %s: %v", name, err)
 		}
-		desc, err := remote.Get(ref, remoteOptions...)
+		desc, err := registry.GetRemote().Get(ref, remoteOptions...)
 		if err == nil && desc != nil && dig == desc.Digest {
 			fmt.Printf("%s image already available on remote registry, skipping push\n", remoteName)
 			return nil
@@ -85,7 +86,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
 		}
 
 		// get the existing image, if any
-		desc, err := remote.Get(ref, remoteOptions...)
+		desc, err := registry.GetRemote().Get(ref, remoteOptions...)
 		if err == nil && desc != nil {
 			if dig == desc.Digest {
 				fmt.Printf("%s index already available on remote registry, skipping push\n", remoteName)

src/cmd/linuxkit/cache/write.go

@@ -18,6 +18,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	log "github.com/sirupsen/logrus"
@@ -71,7 +72,7 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
 		return fmt.Errorf("invalid image name %s: %v", pullImageName, err)
 	}
 
-	desc, err := remote.Get(remoteRef, remoteOptions...)
+	desc, err := registry.GetRemote().Get(remoteRef, remoteOptions...)
 	if err != nil {
 		return fmt.Errorf("error getting manifest for image %s: %v", pullImageName, err)
 	}
@@ -430,7 +431,7 @@ func (p *Provider) ImageInRegistry(ref *reference.Spec, trustedRef, architecture
 		return false, fmt.Errorf("invalid image name %s: %v", image, err)
 	}
 
-	desc, err := remote.Get(remoteRef, remoteOptions...)
+	desc, err := registry.GetRemote().Get(remoteRef, remoteOptions...)
 	if err != nil {
 		log.Debugf("Retrieving image %s returned an error, ignoring: %v", image, err)
 		return false, nil

src/cmd/linuxkit/cache_clean.go

@@ -5,8 +5,8 @@ import (
 	"os"
 
 	namepkg "github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
 	cachepkg "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -58,7 +58,7 @@ func removeImagesFromCache(images map[string]string, p *cachepkg.Provider, publi
 			if err != nil {
 				continue
 			}
-			desc, err := remote.Get(ref)
+			desc, err := registry.GetRemote().Get(ref)
 			if err != nil {
 				log.Debugf("image %s not found in remote registry or error, leaving in cache: %v", name, err)
 				fmt.Fprintf(os.Stderr, "image %s not found in remote registry, leaving in cache", name)

src/cmd/linuxkit/cmd.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
@@ -46,6 +48,7 @@ func newCmd() *cobra.Command {
 		flagQuiet       bool
 		flagVerbose     int
 		flagVerboseName = "verbose"
+		mirrorsRaw      []string
 	)
 	cmd := &cobra.Command{
 		Use:               "linuxkit",
@@ -54,6 +57,36 @@ func newCmd() *cobra.Command {
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			readConfig()
 
+			// convert the provided mirrors to a map
+			for _, m := range mirrorsRaw {
+				if m == "" {
+					continue
+				}
+				parts := strings.SplitN(m, "=", 2)
+				// if no equals sign, use the whole string as the mirror for all registries
+				// not otherwise specified
+				var key, value string
+				if len(parts) == 1 {
+					key = "*"
+					value = parts[0]
+				} else {
+					key = parts[0]
+					value = parts[1]
+				}
+				// value must start with http:// or https://
+				if !strings.HasPrefix(value, "http://") && !strings.HasPrefix(value, "https://") {
+					return fmt.Errorf("mirror %q must start with http:// or https://", value)
+				}
+				// special logic for docker.io because of its odd references
+				if key == "docker.io" || key == "docker.io/" {
+					for _, prefix := range []string{"docker.io", "index.docker.io", "registry-1.docker.io"} {
+						registry.SetProxy(prefix, value)
+					}
+				} else {
+					registry.SetProxy(key, value)
+				}
+			}
+
 			// Set up logging
 			return util.SetupLogging(flagQuiet, flagVerbose, cmd.Flag(flagVerboseName).Changed)
 		},
@@ -69,6 +102,7 @@ func newCmd() *cobra.Command {
 	cmd.AddCommand(versionCmd())
 
 	cmd.PersistentFlags().StringVar(&cacheDir, "cache", defaultLinuxkitCache(), fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))
+	cmd.PersistentFlags().StringArrayVar(&mirrorsRaw, "mirror", nil, "Mirror to use for pulling images, format is <registry>=<mirror>, e.g. docker.io=http://mymirror.io, or just http://mymirror.io for all not otherwise specified; must include protocol. Can be provided multiple times.")
 	cmd.PersistentFlags().BoolVarP(&flagQuiet, "quiet", "q", false, "Quiet execution")
 	cmd.PersistentFlags().IntVarP(&flagVerbose, flagVerboseName, "v", 1, "Verbosity of logging: 0 = quiet, 1 = info, 2 = debug, 3 = trace. Default is info. Setting it explicitly will create structured logging lines.")
 

src/cmd/linuxkit/pkg_remotetag.go

@@ -7,6 +7,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/crane"
 	namepkg "github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -41,11 +42,11 @@ func pkgRemoteTagCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			fromDesc, err := remote.Get(fromRef, remoteOptions...)
+			fromDesc, err := registry.GetRemote().Get(fromRef, remoteOptions...)
 			if err != nil {
 				return fmt.Errorf("error getting manifest for from image %s: %v", fromFullname, err)
 			}
-			toDesc, err := remote.Get(toRef, remoteOptions...)
+			toDesc, err := registry.GetRemote().Get(toRef, remoteOptions...)
 			if err == nil {
 				if toDesc.Digest == fromDesc.Digest {
 					log.Infof("image %s already exists in the registry, identical to %s, skipping", toFullname, fromFullname)
@@ -59,7 +60,7 @@ func pkgRemoteTagCmd() *cobra.Command {
 				if err != nil {
 					return err
 				}
-				finalErr = remote.Tag(toTag, fromDesc, remoteOptions...)
+				finalErr = registry.GetRemote().Tag(toTag, fromDesc, remoteOptions...)
 			} else {
 				// different, so need to copy
 				finalErr = crane.Copy(fromFullname, toFullname)

src/cmd/linuxkit/registry/manifest.go

@@ -48,7 +48,7 @@ func PushManifest(img string, options ...remote.Option) (hash string, length int
 		if err != nil {
 			return hash, length, fmt.Errorf("parsing %s: %w", refName, err)
 		}
-		remoteDesc, err := remote.Get(ref, options...)
+		remoteDesc, err := GetRemote().Get(ref, options...)
 		if err != nil {
 			// TODO: Should distinguish between a 404 and a network error
 			log.Warnf("image %s not found; skipping: %v", ref, err)
@@ -74,7 +74,7 @@ func PushManifest(img string, options ...remote.Option) (hash string, length int
 	index := mutate.AppendManifests(empty.Index, adds...)
 	// base index with which we are working
 	// get the existing index, if any
-	desc, err := remote.Get(baseRef, options...)
+	desc, err := GetRemote().Get(baseRef, options...)
 	if err == nil && desc != nil {
 		ii, err := desc.ImageIndex()
 		if err != nil {

src/cmd/linuxkit/registry/remote.go

@@ -0,0 +1,242 @@
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+)
+
+// proxy is a map of registry names to proxy URLs.
+var proxy = make(map[string]string)
+
+func SetProxy(registry, url string) {
+	if url == "" {
+		delete(proxy, registry)
+	} else {
+		proxy[registry] = url
+	}
+}
+
+// Remote implements the functions of
+// github.com/google/go-containerregistry/pkg/v1/remote, while possibly pre-configured for
+// items like proxies, mirrors, authentication, or other settings.
+type Remote struct {
+	proxy map[string]string
+}
+
+// GetRemote returns a Remote
+func GetRemote() *Remote {
+	return &Remote{
+		proxy: proxy,
+	}
+}
+
+func (r *Remote) Get(ref name.Reference, options ...remote.Option) (*remote.Descriptor, error) {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Get(ref, options...)
+}
+
+func (r *Remote) Head(ref name.Reference, options ...remote.Option) (*v1.Descriptor, error) {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Head(ref, options...)
+}
+
+func (r *Remote) Tag(ref name.Tag, t remote.Taggable, options ...remote.Option) error {
+	return remote.Tag(ref, t, options...)
+}
+
+func (r *Remote) Push(ref name.Reference, t remote.Taggable, options ...remote.Option) error {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Push(ref, t, options...)
+}
+
+func (r *Remote) Put(ref name.Reference, t remote.Taggable, options ...remote.Option) error {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Put(ref, t, options...)
+}
+
+func (r *Remote) Write(ref name.Reference, img v1.Image, options ...remote.Option) error {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Write(ref, img, options...)
+}
+
+func (r *Remote) WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...remote.Option) error {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.WriteIndex(ref, ii, options...)
+}
+
+func (r *Remote) WriteLayer(repo name.Repository, layer v1.Layer, options ...remote.Option) error {
+	var err error
+	repo, err = r.rewriteRepository(repo)
+	if err != nil {
+		return fmt.Errorf("rewriting repository %q: %w", repo.Name(), err)
+	}
+
+	return remote.WriteLayer(repo, layer, options...)
+}
+
+func (r *Remote) List(repo name.Repository, options ...remote.Option) ([]string, error) {
+	var err error
+	repo, err = r.rewriteRepository(repo)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting repository %q: %w", repo.Name(), err)
+	}
+	return remote.List(repo, options...)
+}
+
+func (r *Remote) Layer(ref name.Digest, options ...remote.Option) (v1.Layer, error) {
+	var err error
+	ref, err = r.rewriteDigest(ref)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting digest %q: %w", ref.Name(), err)
+	}
+	return remote.Layer(ref, options...)
+}
+
+func (r *Remote) Index(ref name.Reference, options ...remote.Option) (v1.ImageIndex, error) {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Index(ref, options...)
+}
+
+func (r *Remote) Image(ref name.Reference, options ...remote.Option) (v1.Image, error) {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return nil, fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Image(ref, options...)
+}
+
+func (r *Remote) Delete(ref name.Reference, options ...remote.Option) error {
+	var err error
+	ref, err = r.rewriteReference(ref)
+	if err != nil {
+		return fmt.Errorf("rewriting reference %q: %w", ref.Name(), err)
+	}
+
+	return remote.Delete(ref, options...)
+}
+
+func (r *Remote) rewriteReference(ref name.Reference) (name.Reference, error) {
+	newRepo, opts, err := r.rewriteRepositoryBase(ref.Context())
+	if err != nil {
+		return nil, fmt.Errorf("rewriting repository %q: %w", ref.Context().Name(), err)
+	}
+
+	switch typed := ref.(type) {
+	case name.Tag:
+		return name.NewTag(newRepo+":"+typed.TagStr(), opts...)
+	case name.Digest:
+		return name.NewDigest(newRepo+"@"+typed.DigestStr(), opts...)
+	default:
+		return nil, fmt.Errorf("unsupported reference type: %T", ref)
+	}
+}
+
+func (r *Remote) rewriteRepository(repo name.Repository) (name.Repository, error) {
+	newRepo, opts, err := r.rewriteRepositoryBase(repo)
+	if err != nil {
+		return repo, fmt.Errorf("rewriting repository %q: %w", repo.Name(), err)
+	}
+
+	return name.NewRepository(newRepo, opts...)
+}
+
+func (r *Remote) rewriteDigest(dig name.Digest) (name.Digest, error) {
+	newRepo, opts, err := r.rewriteRepositoryBase(dig.Context())
+	if err != nil {
+		return dig, fmt.Errorf("rewriting repository %q: %w", dig, err)
+	}
+
+	return name.NewDigest(newRepo, opts...)
+}
+
+func (r *Remote) rewriteRepositoryBase(repo name.Repository) (string, []name.Option, error) {
+	originalRegistry := repo.RegistryStr()
+	mirror := r.resolveMirror(originalRegistry)
+
+	// No rewrite needed
+	if mirror == "" || mirror == originalRegistry {
+		return repo.RepositoryStr(), nil, nil
+	}
+
+	// get mirror protocol and separate host+path
+	var (
+		rest     string
+		insecure bool
+		opts     []name.Option
+	)
+
+	switch {
+	case strings.HasPrefix(mirror, "http://"):
+		insecure = true
+		rest = mirror[len("http://"):]
+	case strings.HasPrefix(mirror, "https://"):
+		insecure = false
+		rest = mirror[len("https://"):]
+	default:
+		insecure = false // Default to https if no protocol is specified
+		rest = mirror
+	}
+	if insecure {
+		opts = append(opts, name.Insecure)
+	}
+	opts = append(opts, name.WeakValidation)
+	// Build the new repository: mirror/foo/bar
+	// strip off trailing slash if present, so we do not end up with double slashes
+	newRepo := strings.TrimSuffix(rest, "/") + "/" + repo.RepositoryStr()
+
+	return newRepo, opts, nil
+}
+
+func (r *Remote) resolveMirror(registry string) string {
+	if r.proxy == nil {
+		return registry
+	}
+	if val, ok := r.proxy[registry]; ok {
+		return val
+	}
+	if val, ok := r.proxy["*"]; ok {
+		return val
+	}
+	return registry
+}

test/cases/000_build/080_mirrors/config.yml

@@ -0,0 +1,31 @@
+version: 0.1
+log:
+  level: debug
+  fields:
+    service: registry
+    environment: development
+storage:
+    delete:
+      enabled: true
+    cache:
+        blobdescriptor: inmemory
+    filesystem:
+        rootdirectory: /var/lib/registry
+    tag:
+      concurrencylimit: 5
+http:
+    addr: :5000
+    debug:
+        addr: :5001
+        prometheus:
+            enabled: true
+            path: /metrics
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3
+
+proxy:
+  remoteurl: https://registry-1.docker.io
+  

test/cases/000_build/080_mirrors/test.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# SUMMARY: Check that we go through the mirror when building, and fail if mirror configured but not provided
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+clean_up() {
+	docker kill "${REGISTRY_NAME}" || true
+	[ -n "${CACHEDIR}" ] && rm -rf "${CACHEDIR}"
+	[ -n "${REGISTRY_DIR}" ] && rm -rf "${REGISTRY_DIR}"
+}
+trap clean_up EXIT
+
+# container names
+REGISTRY_NAME="test-registry-$$"
+REGISTRY_DIR=$(mktemp -d)
+CACHEDIR=$(mktemp -d)
+
+
+
+# 2 tests:
+# 1. build a package configured to use a mirror without starting mirror - should fail
+# 2. build a package configured to use a mirror after starting mirror - should succeed
+if linuxkit --mirror http://localhost:5001 --cache ${CACHEDIR} build --format kernel+initrd --name "${NAME}" ./test.yml; then
+  echo "Test 1 failed: build succeeded without starting mirror"
+  exit 1
+fi
+
+# Start registry
+REGISTRY_CID=$(docker run -d --rm -v $(pwd)/config.yml:/etc/distribution/config.yml --name ${REGISTRY_NAME} -p 5001:5000 registry:3)
+
+# this one should succeed
+linuxkit --mirror http://localhost:5001 --cache ${CACHEDIR} build --format kernel+initrd --name "${NAME}" ./test.yml
+
+exit 0

test/cases/000_build/080_mirrors/test.yml

@@ -0,0 +1,10 @@
+kernel:
+  image: linuxkit/kernel:6.6.71
+  cmdline: "console=ttyS0"
+init:
+  - linuxkit/init:8eea386739975a43af558eec757a7dcb3a3d2e7b
+  - linuxkit/runc:667e7ea2c426a2460ca21e3da065a57dbb3369c9
+onboot:
+  - name: dhcpcd
+    image: linuxkit/dhcpcd:157df9ef45a035f1542ec2270e374f18efef98a5
+    command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]

test/cases/040_packages/025_auth/test.sh

@@ -11,7 +11,7 @@ set -e
 clean_up() {
 	docker kill "${REGISTRY_NAME}" || true
 	DOCKER_CONFIG="${DOCKER_CONFIG}" docker buildx rm "${BUILDKIT_NAME}" || true
-	[ -n "${CACHDIR}" ] && rm -rf "${CACHDIR}"
+	[ -n "${CACHEDIR}" ] && rm -rf "${CACHEDIR}"
 	[ -n "${DOCKER_CONFIG}" ] && rm -rf "${DOCKER_CONFIG}"
 	[ -n "${REGISTRY_DIR}" ] && rm -rf "${REGISTRY_DIR}"
 }