Merge pull request #680 from vrothberg/delete-oci

delete: support OCI images

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/VividCortex/ewma v1.1.1 // indirect
 	github.com/containerd/continuity v0.0.0-20180216233310-d8fb8589b0e8 // indirect
 	github.com/containers/buildah v1.8.4
-	github.com/containers/image v2.0.0+incompatible
+	github.com/containers/image v1.5.2-0.20190620105408-93b1deece293
 	github.com/containers/storage v1.12.10
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/containerd/continuity v0.0.0-20180216233310-d8fb8589b0e8 h1:ZZOFPzvZO
 github.com/containerd/continuity v0.0.0-20180216233310-d8fb8589b0e8/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containers/buildah v1.8.4 h1:06c+UNeEWMa2wA1Z7muZ0ZqUzE91sDuZJbB0BiZaeYQ=
 github.com/containers/buildah v1.8.4/go.mod h1:1CsiLJvyU+h+wOjnqJJOWuJCVcMxZOr5HN/gHGdzJxY=
+github.com/containers/image v1.5.2-0.20190620105408-93b1deece293 h1:EalCgZ875kDCN2HcOch50q48GKerWGc5eV0BllCvln8=
+github.com/containers/image v1.5.2-0.20190620105408-93b1deece293/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk=
 github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/storage v1.12.10 h1:vw1aiLsZ1LvO09ELMxVBTe35tThRiMftI2cPeH+G5ow=

vendor.conf

@@ -1,67 +0,0 @@
-
-github.com/urfave/cli v1.20.0
-github.com/kr/pretty v0.1.0
-github.com/kr/text v0.1.0
-github.com/containers/image v2.0.0
-github.com/containers/buildah v1.8.4
-github.com/vbauerster/mpb v3.3.4
-github.com/mattn/go-isatty v0.0.4
-github.com/VividCortex/ewma v1.1.1
-golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e
-github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7
-github.com/containers/storage v1.12.10
-github.com/sirupsen/logrus v1.0.0
-github.com/go-check/check v1
-github.com/stretchr/testify v1.1.3
-github.com/davecgh/go-spew v1.1.1
-github.com/pmezard/go-difflib 5d4384ee4fb2527b0a1256a821ebfc92f91efefc
-github.com/pkg/errors v0.8.1
-golang.org/x/crypto a4c6cb3142f211c99e4bf4cd769535b29a9b616f
-github.com/ulikunitz/xz v0.5.4
-github.com/etcd-io/bbolt v1.3.2
-# docker deps from https://github.com/docker/docker/blob/v1.11.2/hack/vendor.sh
-github.com/docker/docker da99009bbb1165d1ac5688b5c81d2f589d418341
-github.com/docker/go-connections 7beb39f0b969b075d1325fecb092faf27fd357b6
-github.com/containerd/continuity d8fb8589b0e8e85b8c8bbaa8840226d0dfeb7371
-github.com/vbatts/tar-split v0.10.2
-github.com/gorilla/context 14f550f51a
-github.com/gorilla/mux e444e69cbd
-github.com/docker/go-units 8a7beacffa3009a9ac66bad506b18ffdd110cf97
-golang.org/x/net 45ffb0cd1ba084b73e26dee67e667e1be5acce83
-github.com/gogo/protobuf fcdc5011193ff531a548e9b0301828d5a5b97fd8
-# end docker deps
-golang.org/x/text e6919f6577db79269a6443b9dc46d18f2238fb5d
-github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716
-# docker/distributions dependencies
-# end of docker/distribution dependencies
-github.com/docker/libtrust aabc10ec26b754e797f9028f4589c5b7bd90dc20
-github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
-github.com/opencontainers/runc v1.0.0-rc6
-github.com/opencontainers/image-spec 7b1e489870acb042978a3935d2fb76f8a79aff81
-# -- start OCI image validation requirements.
-github.com/opencontainers/runtime-spec v1.0.0
-github.com/opencontainers/image-tools 6d941547fa1df31900990b3fb47ec2468c9c6469
-github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
-github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
-github.com/xeipuuv/gojsonschema v1.1.0
-go4.org ce4c26f7be8eb27dc77f996b08d286dd80bc4a01 https://github.com/camlistore/go4
-github.com/ostreedev/ostree-go 56f3a639dbc0f2f5051c6d52dade28a882ba78ce
-# -- end OCI image validation requirements
-github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9
-# openshift/origin' k8s dependencies as of OpenShift v1.1.5
-k8s.io/client-go kubernetes-1.10.13-beta.0
-github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee
-gopkg.in/yaml.v2 d466437aa4adc35830964cffc5b5f262c63ddcb4
-github.com/imdario/mergo 6633656539c1639d9d78127b7d47c622b5d7b6dc
-# containers/storage's dependencies that aren't already being pulled in
-github.com/mistifyio/go-zfs 22c9b32c84eb0d0c6f4043b6e90fc94073de92fa
-github.com/pborman/uuid v1.0
-github.com/opencontainers/selinux v1.1
-golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08
-github.com/tchap/go-patricia v2.2.6
-github.com/BurntSushi/toml v0.3.1
-github.com/pquerna/ffjson d49c2bc1aa135aad0c6f4fc2056623ec78f5d5ac
-github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2
-github.com/klauspost/pgzip v1.2.1
-github.com/klauspost/compress v1.4.1
-github.com/klauspost/cpuid v1.2.0

vendor/github.com/containers/image/docker/docker_image_src.go

@@ -138,8 +138,9 @@ func (s *dockerImageSource) GetManifest(ctx context.Context, instanceDigest *dig
 
 func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest string) ([]byte, string, error) {
 	path := fmt.Sprintf(manifestPath, reference.Path(s.ref.ref), tagOrDigest)
-	headers := make(map[string][]string)
-	headers["Accept"] = manifest.DefaultRequestedManifestMIMETypes
+	headers := map[string][]string{
+		"Accept": manifest.DefaultRequestedManifestMIMETypes,
+	}
 	res, err := s.c.makeRequest(ctx, "GET", path, headers, nil, v2Auth, nil)
 	if err != nil {
 		return nil, "", err
@@ -381,11 +382,9 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere
 		return err
 	}
 
-	// When retrieving the digest from a registry >= 2.3 use the following header:
-	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
-	headers := make(map[string][]string)
-	headers["Accept"] = []string{manifest.DockerV2Schema2MediaType}
-
+	headers := map[string][]string{
+		"Accept": manifest.DefaultRequestedManifestMIMETypes,
+	}
 	refTail, err := ref.tagOrDigest()
 	if err != nil {
 		return err

vendor/github.com/containers/image/docker/tarfile/src.go

@@ -15,24 +15,24 @@ import (
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/types"
-	"github.com/opencontainers/go-digest"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
 
 // Source is a partial implementation of types.ImageSource for reading from tarPath.
 type Source struct {
 	tarPath              string
-	removeTarPathOnClose bool      // Remove temp file on close if true
-	cacheDataLock        sync.Once // Atomic way to ensure that ensureCachedDataIsPresent is only invoked once
+	removeTarPathOnClose bool // Remove temp file on close if true
 	// The following data is only available after ensureCachedDataIsPresent() succeeds
-	cacheDataResult   error         // The return value of ensureCachedDataIsPresent, since it should be as safe to cache as the side effects
 	tarManifest       *ManifestItem // nil if not available yet.
 	configBytes       []byte
 	configDigest      digest.Digest
 	orderedDiffIDList []digest.Digest
 	knownLayers       map[digest.Digest]*layerInfo
 	// Other state
-	generatedManifest []byte // Private cache for GetManifest(), nil if not set yet.
+	generatedManifest []byte    // Private cache for GetManifest(), nil if not set yet.
+	cacheDataLock     sync.Once // Private state for ensureCachedDataIsPresent to make it concurrency-safe
+	cacheDataResult   error     // Private state for ensureCachedDataIsPresent
 }
 
 type layerInfo struct {
@@ -201,49 +201,52 @@ func (s *Source) readTarComponent(path string) ([]byte, error) {
 }
 
 // ensureCachedDataIsPresent loads data necessary for any of the public accessors.
+// It is safe to call this from multi-threaded code.
 func (s *Source) ensureCachedDataIsPresent() error {
 	s.cacheDataLock.Do(func() {
-		// Read and parse manifest.json
-		tarManifest, err := s.loadTarManifest()
-		if err != nil {
-			s.cacheDataResult = err
-			return
-		}
-
-		// Check to make sure length is 1
-		if len(tarManifest) != 1 {
-			s.cacheDataResult = errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest))
-			return
-		}
-
-		// Read and parse config.
-		configBytes, err := s.readTarComponent(tarManifest[0].Config)
-		if err != nil {
-			s.cacheDataResult = err
-			return
-		}
-		var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs.
-		if err := json.Unmarshal(configBytes, &parsedConfig); err != nil {
-			s.cacheDataResult = errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config)
-			return
-		}
-
-		knownLayers, err := s.prepareLayerData(&tarManifest[0], &parsedConfig)
-		if err != nil {
-			s.cacheDataResult = err
-			return
-		}
-
-		// Success; commit.
-		s.tarManifest = &tarManifest[0]
-		s.configBytes = configBytes
-		s.configDigest = digest.FromBytes(configBytes)
-		s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs
-		s.knownLayers = knownLayers
+		s.cacheDataResult = s.ensureCachedDataIsPresentPrivate()
 	})
 	return s.cacheDataResult
 }
 
+// ensureCachedDataIsPresentPrivate is a private implementation detail of ensureCachedDataIsPresent.
+// Call ensureCachedDataIsPresent instead.
+func (s *Source) ensureCachedDataIsPresentPrivate() error {
+	// Read and parse manifest.json
+	tarManifest, err := s.loadTarManifest()
+	if err != nil {
+		return err
+	}
+
+	// Check to make sure length is 1
+	if len(tarManifest) != 1 {
+		return errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest))
+	}
+
+	// Read and parse config.
+	configBytes, err := s.readTarComponent(tarManifest[0].Config)
+	if err != nil {
+		return err
+	}
+	var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs.
+	if err := json.Unmarshal(configBytes, &parsedConfig); err != nil {
+		return errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config)
+	}
+
+	knownLayers, err := s.prepareLayerData(&tarManifest[0], &parsedConfig)
+	if err != nil {
+		return err
+	}
+
+	// Success; commit.
+	s.tarManifest = &tarManifest[0]
+	s.configBytes = configBytes
+	s.configDigest = digest.FromBytes(configBytes)
+	s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs
+	s.knownLayers = knownLayers
+	return nil
+}
+
 // loadTarManifest loads and decodes the manifest.json.
 func (s *Source) loadTarManifest() ([]ManifestItem, error) {
 	// FIXME? Do we need to deal with the legacy format?

vendor/github.com/containers/image/ostree/ostree_dest.go

@@ -218,7 +218,7 @@ func fixFiles(selinuxHnd *C.struct_selabel_handle, root string, dir string, user
 				defer C.free(unsafe.Pointer(fullpathC))
 				res, err = C.lsetfilecon_raw(fullpathC, context)
 				if int(res) < 0 {
-					return errors.Wrapf(err, "cannot setfilecon_raw %s", fullpath)
+					return errors.Wrapf(err, "cannot setfilecon_raw %s to %s", fullpath, C.GoString(context))
 				}
 			}
 		}

vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go

@@ -30,10 +30,10 @@ const builtinRegistriesConfPath = "/etc/containers/registries.conf"
 // Endpoint describes a remote location of a registry.
 type Endpoint struct {
 	// The endpoint's remote location.
-	Location string `toml:"location"`
+	Location string `toml:"location,omitempty"`
 	// If true, certs verification will be skipped and HTTP (non-TLS)
 	// connections will be allowed.
-	Insecure bool `toml:"insecure"`
+	Insecure bool `toml:"insecure,omitempty"`
 }
 
 // rewriteReference will substitute the provided reference `prefix` to the
@@ -56,22 +56,22 @@ func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (referen
 
 // Registry represents a registry.
 type Registry struct {
-	// A registry is an Endpoint too
-	Endpoint
-	// The registry's mirrors.
-	Mirrors []Endpoint `toml:"mirror"`
-	// If true, pulling from the registry will be blocked.
-	Blocked bool `toml:"blocked"`
-	// If true, mirrors will only be used for digest pulls. Pulling images by
-	// tag can potentially yield different images, depending on which endpoint
-	// we pull from.  Forcing digest-pulls for mirrors avoids that issue.
-	MirrorByDigestOnly bool `toml:"mirror-by-digest-only"`
 	// Prefix is used for matching images, and to translate one namespace to
 	// another.  If `Prefix="example.com/bar"`, `location="example.com/foo/bar"`
 	// and we pull from "example.com/bar/myimage:latest", the image will
 	// effectively be pulled from "example.com/foo/bar/myimage:latest".
 	// If no Prefix is specified, it defaults to the specified location.
 	Prefix string `toml:"prefix"`
+	// A registry is an Endpoint too
+	Endpoint
+	// The registry's mirrors.
+	Mirrors []Endpoint `toml:"mirror,omitempty"`
+	// If true, pulling from the registry will be blocked.
+	Blocked bool `toml:"blocked,omitempty"`
+	// If true, mirrors will only be used for digest pulls. Pulling images by
+	// tag can potentially yield different images, depending on which endpoint
+	// we pull from.  Forcing digest-pulls for mirrors avoids that issue.
+	MirrorByDigestOnly bool `toml:"mirror-by-digest-only,omitempty"`
 }
 
 // PullSource consists of an Endpoint and a Reference. Note that the reference is

vendor/github.com/containers/image/storage/storage_image.go

@@ -491,14 +491,21 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t
 
 	// Does the blob correspond to a known DiffID which we already have available?
 	// Because we must return the size, which is unknown for unavailable compressed blobs, the returned BlobInfo refers to the
-	// uncompressed layer, and that can happen only if canSubstitute.
-	if canSubstitute {
+	// uncompressed layer, and that can happen only if canSubstitute, or if the incoming manifest already specifies the size.
+	if canSubstitute || blobinfo.Size != -1 {
 		if uncompressedDigest := cache.UncompressedDigest(blobinfo.Digest); uncompressedDigest != "" && uncompressedDigest != blobinfo.Digest {
 			layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest)
 			if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
 				return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for layers with digest %q`, uncompressedDigest)
 			}
 			if len(layers) > 0 {
+				if blobinfo.Size != -1 {
+					s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest
+					return true, blobinfo, nil
+				}
+				if !canSubstitute {
+					return false, types.BlobInfo{}, fmt.Errorf("Internal error: canSubstitute was expected to be true for blobInfo %v", blobinfo)
+				}
 				s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest
 				return true, types.BlobInfo{
 					Digest:    uncompressedDigest,
@@ -627,7 +634,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
 		if !ok {
 			// Try to find the layer with contents matching that blobsum.
 			layer := ""
-			layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(blob.Digest)
+			layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID)
 			if err2 == nil && len(layers) > 0 {
 				layer = layers[0].ID
 			} else {

vendor/github.com/containers/image/version/version.go

@@ -8,10 +8,10 @@ const (
 	// VersionMinor is for functionality in a backwards-compatible manner
 	VersionMinor = 0
 	// VersionPatch is for backwards-compatible bug fixes
-	VersionPatch = 0
+	VersionPatch = 1
 
 	// VersionDev indicates development branch. Releases will be empty string.
-	VersionDev = ""
+	VersionDev = "-dev"
 )
 
 // Version is the specification version that the package types support.

vendor/modules.txt

@@ -26,7 +26,7 @@ github.com/VividCortex/ewma
 github.com/containerd/continuity/pathdriver
 # github.com/containers/buildah v1.8.4
 github.com/containers/buildah/pkg/unshare
-# github.com/containers/image v2.0.0+incompatible
+# github.com/containers/image v1.5.2-0.20190620105408-93b1deece293
 github.com/containers/image/copy
 github.com/containers/image/directory
 github.com/containers/image/docker