skopeo-copy: docker-archive: multitag support

Add multitag support when generating docker-archive tarballs via the
newly added '--aditional-tag' option, which can be specified multiple
times to add more than one tag.  All specified tags will be added to the
RepoTags field in the docker-archive's manifest.json file.

This change requires to vendor the latest containers/image with
commit a1a9391830fd08637edbe45133fd0a8a2682ae75.

Signed-off-by: Valentin Rothberg <vrothberg@suse.com>

cmd/skopeo/copy.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/containers/image/copy"
+	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/transports"
 	"github.com/containers/image/transports/alltransports"
@@ -72,6 +73,20 @@ func copyHandler(c *cli.Context) error {
 		}
 	}
 
+	if c.IsSet("additional-tag") {
+		for _, image := range c.StringSlice("additional-tag") {
+			ref, err := reference.ParseNormalizedNamed(image)
+			if err != nil {
+				return fmt.Errorf("error parsing additional-tag '%s': %v", image, err)
+			}
+			namedTagged, isNamedTagged := ref.(reference.NamedTagged)
+			if !isNamedTagged {
+				return fmt.Errorf("additional-tag '%s' must be a tagged reference", image)
+			}
+			destinationCtx.DockerArchiveAdditionalTags = append(destinationCtx.DockerArchiveAdditionalTags, namedTagged)
+		}
+	}
+
 	return copy.Image(context.Background(), policyContext, destRef, srcRef, &copy.Options{
 		RemoveSignatures:      removeSignatures,
 		SignBy:                signBy,
@@ -98,6 +113,10 @@ var copyCmd = cli.Command{
 	Action:    copyHandler,
 	// FIXME: Do we need to namespace the GPG aspect?
 	Flags: []cli.Flag{
+		cli.StringSliceFlag{
+			Name:  "additional-tag",
+			Usage: "additional tags (supports docker-archive)",
+		},
 		cli.StringFlag{
 			Name:  "authfile",
 			Usage: "path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json",

vendor/github.com/containers/image/docker/archive/dest.go

@@ -16,7 +16,7 @@ type archiveImageDestination struct {
 	writer               io.Closer
 }
 
-func newImageDestination(ref archiveReference) (types.ImageDestination, error) {
+func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) {
 	if ref.destinationRef == nil {
 		return nil, errors.Errorf("docker-archive: destination reference not supplied (must be of form <path>:<reference:tag>)")
 	}
@@ -40,8 +40,12 @@ func newImageDestination(ref archiveReference) (types.ImageDestination, error) {
 		return nil, errors.New("docker-archive doesn't support modifying existing images")
 	}
 
+	tarDest := tarfile.NewDestination(fh, ref.destinationRef)
+	if sys != nil && sys.DockerArchiveAdditionalTags != nil {
+		tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags)
+	}
 	return &archiveImageDestination{
-		Destination: tarfile.NewDestination(fh, ref.destinationRef),
+		Destination: tarDest,
 		ref:         ref,
 		writer:      fh,
 	}, nil

vendor/github.com/containers/image/docker/archive/transport.go

@@ -148,7 +148,7 @@ func (ref archiveReference) NewImageSource(ctx context.Context, sys *types.Syste
 // NewImageDestination returns a types.ImageDestination for this reference.
 // The caller must call .Close() on the returned ImageDestination.
 func (ref archiveReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
-	return newImageDestination(ref)
+	return newImageDestination(sys, ref)
 }
 
 // DeleteImage deletes the named image from the registry, if supported.

vendor/github.com/containers/image/docker/daemon/daemon_transport.go

@@ -2,13 +2,15 @@ package daemon
 
 import (
 	"context"
-	"github.com/pkg/errors"
+	"fmt"
 
+	"github.com/containers/image/docker/policyconfiguration"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/image"
 	"github.com/containers/image/transports"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -35,8 +37,15 @@ func (t daemonTransport) ParseReference(reference string) (types.ImageReference,
 // It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion.
 // scope passed to this function will not be "", that value is always allowed.
 func (t daemonTransport) ValidatePolicyConfigurationScope(scope string) error {
-	// See the explanation in daemonReference.PolicyConfigurationIdentity.
+	// ID values cannot be effectively namespaced, and are clearly invalid host:port values.
-	return errors.New(`docker-daemon: does not support any scopes except the default "" one`)
+	if _, err := digest.Parse(scope); err == nil {
+		return errors.Errorf(`docker-daemon: can not use algo:digest value %s as a namespace`, scope)
+	}
+
+	// FIXME? We could be verifying the various character set and length restrictions
+	// from docker/distribution/reference.regexp.go, but other than that there
+	// are few semantically invalid strings.
+	return nil
 }
 
 // daemonReference is an ImageReference for images managed by a local Docker daemon
@@ -88,6 +97,8 @@ func NewReference(id digest.Digest, ref reference.Named) (types.ImageReference,
 		// A github.com/distribution/reference value can have a tag and a digest at the same time!
 		// Most versions of docker/reference do not handle that (ignoring the tag), so reject such input.
 		// This MAY be accepted in the future.
+		// (Even if it were supported, the semantics of policy namespaces are unclear - should we drop
+		// the tag or the digest first?)
 		_, isTagged := ref.(reference.NamedTagged)
 		_, isDigested := ref.(reference.Canonical)
 		if isTagged && isDigested {
@@ -137,9 +148,28 @@ func (ref daemonReference) DockerReference() reference.Named {
 // Returns "" if configuration identities for these references are not supported.
 func (ref daemonReference) PolicyConfigurationIdentity() string {
 	// We must allow referring to images in the daemon by image ID, otherwise untagged images would not be accessible.
-	// But the existence of image IDs means that we can’t truly well namespace the input; the untagged images would have to fall into the default policy,
+	// But the existence of image IDs means that we can’t truly well namespace the input:
-	// which can be unexpected.  So, punt.
+	// a single image can be namespaced either using the name or the ID depending on how it is named.
-	return "" // This still allows using the default "" scope to define a policy for this transport.
+	//
+	// That’s fairly unexpected, but we have to cope somehow.
+	//
+	// So, use the ordinary docker/policyconfiguration namespacing for named images.
+	// image IDs all fall into the root namespace.
+	// Users can set up the root namespace to be either untrusted or rejected,
+	// and to set up specific trust for named namespaces.  This allows verifying image
+	// identity when a name is known, and unnamed images would be untrusted or rejected.
+	switch {
+	case ref.id != "":
+		return "" // This still allows using the default "" scope to define a global policy for ID-identified images.
+	case ref.ref != nil:
+		res, err := policyconfiguration.DockerReferenceIdentity(ref.ref)
+		if res == "" || err != nil { // Coverage: Should never happen, NewReference above should refuse values which could cause a failure.
+			panic(fmt.Sprintf("Internal inconsistency: policyconfiguration.DockerReferenceIdentity returned %#v, %v", res, err))
+		}
+		return res
+	default: // Coverage: Should never happen, NewReference above should refuse such values.
+		panic("Internal inconsistency: daemonReference has empty id and nil ref")
+	}
 }
 
 // PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
@@ -149,7 +179,14 @@ func (ref daemonReference) PolicyConfigurationIdentity() string {
 // and each following element to be a prefix of the element preceding it.
 func (ref daemonReference) PolicyConfigurationNamespaces() []string {
 	// See the explanation in daemonReference.PolicyConfigurationIdentity.
-	return []string{}
+	switch {
+	case ref.id != "":
+		return []string{}
+	case ref.ref != nil:
+		return policyconfiguration.DockerReferenceNamespaces(ref.ref)
+	default: // Coverage: Should never happen, NewReference above should refuse such values.
+		panic("Internal inconsistency: daemonReference has empty id and nil ref")
+	}
 }
 
 // NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.

vendor/github.com/containers/image/docker/docker_image.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
 
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/image"
@@ -39,25 +41,67 @@ func (i *Image) SourceRefFullName() string {
 	return i.src.ref.ref.Name()
 }
 
-// GetRepositoryTags list all tags available in the repository. Note that this has no connection with the tag(s) used for this specific image, if any.
+// GetRepositoryTags list all tags available in the repository. The tag
+// provided inside the ImageReference will be ignored. (This is a
+// backward-compatible shim method which calls the module-level
+// GetRepositoryTags)
 func (i *Image) GetRepositoryTags(ctx context.Context) ([]string, error) {
-	path := fmt.Sprintf(tagsPath, reference.Path(i.src.ref.ref))
+	return GetRepositoryTags(ctx, i.src.c.sys, i.src.ref)
-	// FIXME: Pass the context.Context
+}
-	res, err := i.src.c.makeRequest(ctx, "GET", path, nil, nil)
+
-	if err != nil {
+// GetRepositoryTags list all tags available in the repository. The tag
-		return nil, err
+// provided inside the ImageReference will be ignored.
-	}
+func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types.ImageReference) ([]string, error) {
-	defer res.Body.Close()
+	dr, ok := ref.(dockerReference)
-	if res.StatusCode != http.StatusOK {
+	if !ok {
-		// print url also
+		return nil, errors.Errorf("ref must be a dockerReference")
-		return nil, errors.Errorf("Invalid status code returned when fetching tags list %d", res.StatusCode)
+	}
-	}
+
-	type tagsRes struct {
+	path := fmt.Sprintf(tagsPath, reference.Path(dr.ref))
-		Tags []string
+	client, err := newDockerClientFromRef(sys, dr, false, "pull")
-	}
+	if err != nil {
-	tags := &tagsRes{}
+		return nil, errors.Wrap(err, "failed to create client")
-	if err := json.NewDecoder(res.Body).Decode(tags); err != nil {
+	}
-		return nil, err
+
-	}
+	tags := make([]string, 0)
-	return tags.Tags, nil
+
+	for {
+		res, err := client.makeRequest(ctx, "GET", path, nil, nil)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusOK {
+			// print url also
+			return nil, errors.Errorf("Invalid status code returned when fetching tags list %d", res.StatusCode)
+		}
+
+		var tagsHolder struct {
+			Tags []string
+		}
+		if err = json.NewDecoder(res.Body).Decode(&tagsHolder); err != nil {
+			return nil, err
+		}
+		tags = append(tags, tagsHolder.Tags...)
+
+		link := res.Header.Get("Link")
+		if link == "" {
+			break
+		}
+
+		linkURLStr := strings.Trim(strings.Split(link, ";")[0], "<>")
+		linkURL, err := url.Parse(linkURLStr)
+		if err != nil {
+			return tags, err
+		}
+
+		// can be relative or absolute, but we only want the path (and I
+		// guess we're in trouble if it forwards to a new place...)
+		path = linkURL.Path
+		if linkURL.RawQuery != "" {
+			path += "?"
+			path += linkURL.RawQuery
+		}
+	}
+	return tags, nil
 }

vendor/github.com/containers/image/docker/tarfile/dest.go

@@ -26,6 +26,7 @@ type Destination struct {
 	writer    io.Writer
 	tar       *tar.Writer
 	reference reference.NamedTagged
+	repoTags  []reference.NamedTagged
 	// Other state.
 	blobs  map[digest.Digest]types.BlobInfo // list of already-sent blobs
 	config []byte
@@ -37,10 +38,16 @@ func NewDestination(dest io.Writer, ref reference.NamedTagged) *Destination {
 		writer:    dest,
 		tar:       tar.NewWriter(dest),
 		reference: ref,
+		repoTags:  []reference.NamedTagged{ref},
 		blobs:     make(map[digest.Digest]types.BlobInfo),
 	}
 }
 
+// AddRepoTags adds the specified tags to the destination's repoTags.
+func (d *Destination) AddRepoTags(tags []reference.NamedTagged) {
+	d.repoTags = append(d.repoTags, tags...)
+}
+
 // SupportedManifestMIMETypes tells which manifest mime types the destination supports
 // If an empty slice or nil it's returned, then any mime type can be tried to upload
 func (d *Destination) SupportedManifestMIMETypes() []string {
@@ -199,26 +206,31 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte) error {
 		}
 	}
 
-	// For github.com/docker/docker consumers, this works just as well as
+	repoTags := []string{}
-	//   refString := ref.String()
+	for _, tag := range d.repoTags {
-	// because when reading the RepoTags strings, github.com/docker/docker/reference
+		// For github.com/docker/docker consumers, this works just as well as
-	// normalizes both of them to the same value.
+		//   refString := ref.String()
-	//
+		// because when reading the RepoTags strings, github.com/docker/docker/reference
-	// Doing it this way to include the normalized-out `docker.io[/library]` does make
+		// normalizes both of them to the same value.
-	// a difference for github.com/projectatomic/docker consumers, with the
+		//
-	// “Add --add-registry and --block-registry options to docker daemon” patch.
+		// Doing it this way to include the normalized-out `docker.io[/library]` does make
-	// These consumers treat reference strings which include a hostname and reference
+		// a difference for github.com/projectatomic/docker consumers, with the
-	// strings without a hostname differently.
+		// “Add --add-registry and --block-registry options to docker daemon” patch.
-	//
+		// These consumers treat reference strings which include a hostname and reference
-	// Using the host name here is more explicit about the intent, and it has the same
+		// strings without a hostname differently.
-	// effect as (docker pull) in projectatomic/docker, which tags the result using
+		//
-	// a hostname-qualified reference.
+		// Using the host name here is more explicit about the intent, and it has the same
-	// See https://github.com/containers/image/issues/72 for a more detailed
+		// effect as (docker pull) in projectatomic/docker, which tags the result using
-	// analysis and explanation.
+		// a hostname-qualified reference.
-	refString := fmt.Sprintf("%s:%s", d.reference.Name(), d.reference.Tag())
+		// See https://github.com/containers/image/issues/72 for a more detailed
+		// analysis and explanation.
+		refString := fmt.Sprintf("%s:%s", tag.Name(), tag.Tag())
+		repoTags = append(repoTags, refString)
+	}
+
 	items := []ManifestItem{{
 		Config:       man.ConfigDescriptor.Digest.Hex() + ".json",
-		RepoTags:     []string{refString},
+		RepoTags:     repoTags,
 		Layers:       layerPaths,
 		Parent:       "",
 		LayerSources: nil,

vendor/github.com/containers/image/internal/tmpdir/tmpdir.go

@@ -5,6 +5,16 @@ import (
 	"runtime"
 )
 
+// unixTempDirForBigFiles is the directory path to store big files on non Windows systems.
+// You can override this at build time with
+// -ldflags '-X github.com/containers/image/internal/tmpdir.unixTempDirForBigFiles=$your_path'
+var unixTempDirForBigFiles = builtinUnixTempDirForBigFiles
+
+// builtinUnixTempDirForBigFiles is the directory path to store big files.
+// Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs.
+// DO NOT change this, instead see unixTempDirForBigFiles above.
+const builtinUnixTempDirForBigFiles = "/var/tmp"
+
 // TemporaryDirectoryForBigFiles returns a directory for temporary (big) files.
 // On non Windows systems it avoids the use of os.TempDir(), because the default temporary directory usually falls under /tmp
 // which on systemd based systems could be the unsuitable tmpfs filesystem.
@@ -13,7 +23,7 @@ func TemporaryDirectoryForBigFiles() string {
 	if runtime.GOOS == "windows" {
 		temporaryDirectoryForBigFiles = os.TempDir()
 	} else {
-		temporaryDirectoryForBigFiles = "/var/tmp"
+		temporaryDirectoryForBigFiles = unixTempDirForBigFiles
 	}
 	return temporaryDirectoryForBigFiles
 }

vendor/github.com/containers/image/pkg/docker/config/config.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker-credential-helpers/credentials"
 	"github.com/docker/docker/pkg/homedir"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type dockerAuthConfig struct {
@@ -64,11 +65,17 @@ func GetAuthentication(sys *types.SystemContext, registry string) (string, strin
 	}
 
 	dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyCfg)
+	var paths []string
 	pathToAuth, err := getPathToAuth(sys)
-	if err != nil {
+	if err == nil {
-		return "", "", err
+		paths = append(paths, pathToAuth)
+	} else {
+		// Error means that the path set for XDG_RUNTIME_DIR does not exist
+		// but we don't want to completely fail in the case that the user is pulling a public image
+		// Logging the error as a warning instead and moving on to pulling the image
+		logrus.Warnf("%v: Trying to pull image in the event that it is a public image.", err)
 	}
-	paths := [3]string{pathToAuth, filepath.Join(homedir.Get(), dockerCfg, dockerCfgFileName), dockerLegacyPath}
+	paths = append(paths, filepath.Join(homedir.Get(), dockerCfg, dockerCfgFileName), dockerLegacyPath)
 
 	for _, path := range paths {
 		legacyFormat := path == dockerLegacyPath

vendor/github.com/containers/image/storage/storage_image.go

@@ -14,6 +14,7 @@ import (
 	"sync/atomic"
 
 	"github.com/containers/image/image"
+	"github.com/containers/image/internal/tmpdir"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/containers/storage"
@@ -25,8 +26,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs.
-
 var (
 	// ErrBlobDigestMismatch is returned when PutBlob() is given a blob
 	// with a digest-based name that doesn't match its contents.
@@ -240,7 +239,7 @@ func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *
 // newImageDestination sets us up to write a new image, caching blobs in a temporary directory until
 // it's time to Commit() the image
 func newImageDestination(imageRef storageReference) (*storageImageDestination, error) {
-	directory, err := ioutil.TempDir(temporaryDirectoryForBigFiles, "storage")
+	directory, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(), "storage")
 	if err != nil {
 		return nil, errors.Wrapf(err, "error creating a temporary directory")
 	}

vendor/github.com/containers/image/types/types.go

@@ -347,6 +347,9 @@ type SystemContext struct {
 	// If not "", overrides the use of platform.GOOS when choosing an image or verifying OS match.
 	OSChoice string
 
+	// Additional tags when creating or copying a docker-archive.
+	DockerArchiveAdditionalTags []reference.NamedTagged
+
 	// === OCI.Transport overrides ===
 	// If not "", a directory containing a CA certificate (ending with ".crt"),
 	// a client certificate (ending with ".cert") and a client ceritificate key

vendor/github.com/containers/image/vendor.conf

@@ -10,11 +10,9 @@ github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d
 github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52
 github.com/docker/libtrust aabc10ec26b754e797f9028f4589c5b7bd90dc20
 github.com/ghodss/yaml 04f313413ffd65ce25f2541bfd2b2ceec5c0908c
-github.com/gorilla/context 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
 github.com/gorilla/mux 94e7d24fd285520f3d12ae998f7fdd6b5393d453
 github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade
 github.com/mattn/go-runewidth 14207d285c6c197daabb5c9793d63e7af9ab2d50
-github.com/mattn/go-shellwords 005a0944d84452842197c2108bd9168ced206f78
 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9
 github.com/opencontainers/go-digest aa2ec055abd10d26d539eb630a92241b781ce4bc