Merge pull request #112 from runcom/manifest-pkg

move manifests stuff to its own pkg and add OCI mime types

cmd/skopeo/copy.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/signature"
 	"github.com/urfave/cli"
 )
@@ -56,12 +56,12 @@ func copyHandler(context *cli.Context) error {
 	}
 	signBy := context.String("sign-by")
 
-	manifest, _, err := src.GetManifest([]string{utils.DockerV2Schema1MIMEType})
+	m, _, err := src.GetManifest([]string{manifest.DockerV2Schema1MIMEType})
 	if err != nil {
 		return fmt.Errorf("Error reading manifest: %v", err)
 	}
 
-	layers, err := manifestLayers(manifest)
+	layers, err := manifestLayers(m)
 	if err != nil {
 		return fmt.Errorf("Error parsing manifest: %v", err)
 	}
@@ -92,7 +92,7 @@ func copyHandler(context *cli.Context) error {
 			return fmt.Errorf("Error determining canonical Docker reference: %v", err)
 		}
 
-		newSig, err := signature.SignDockerManifest(manifest, dockerReference, mech, signBy)
+		newSig, err := signature.SignDockerManifest(m, dockerReference, mech, signBy)
 		if err != nil {
 			return fmt.Errorf("Error creating signature: %v", err)
 		}
@@ -104,7 +104,7 @@ func copyHandler(context *cli.Context) error {
 	}
 
 	// FIXME: We need to call PutManifest after PutBlob and PutSignatures. This seems ugly; move to a "set properties" + "commit" model?
-	if err := dest.PutManifest(manifest); err != nil {
+	if err := dest.PutManifest(m); err != nil {
 		return fmt.Errorf("Error writing manifest: %v", err)
 	}
 	return nil

cmd/skopeo/inspect.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/projectatomic/skopeo/docker"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/urfave/cli"
 )
 
@@ -62,7 +62,7 @@ var inspectCmd = cli.Command{
 			Os:            imgInspect.Os,
 			Layers:        imgInspect.Layers,
 		}
-		outputData.Digest, err = utils.ManifestDigest(rawManifest)
+		outputData.Digest, err = manifest.Digest(rawManifest)
 		if err != nil {
 			return fmt.Errorf("Error computing manifest digest: %v", err)
 		}

docker/docker_image_dest.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/reference"
 	"github.com/projectatomic/skopeo/types"
 )
@@ -40,21 +40,21 @@ func (d *dockerImageDestination) CanonicalDockerReference() (string, error) {
 	return fmt.Sprintf("%s:%s", d.ref.Name(), d.tag), nil
 }
 
-func (d *dockerImageDestination) PutManifest(manifest []byte) error {
+func (d *dockerImageDestination) PutManifest(m []byte) error {
 	// FIXME: This only allows upload by digest, not creating a tag.  See the
 	// corresponding comment in NewOpenshiftImageDestination.
-	digest, err := utils.ManifestDigest(manifest)
+	digest, err := manifest.Digest(m)
 	if err != nil {
 		return err
 	}
 	url := fmt.Sprintf(manifestURL, d.ref.RemoteName(), digest)
 
 	headers := map[string][]string{}
-	mimeType := utils.GuessManifestMIMEType(manifest)
+	mimeType := manifest.GuessMIMEType(m)
 	if mimeType != "" {
 		headers["Content-Type"] = []string{mimeType}
 	}
-	res, err := d.c.makeRequest("PUT", url, headers, bytes.NewReader(manifest))
+	res, err := d.c.makeRequest("PUT", url, headers, bytes.NewReader(m))
 	if err != nil {
 		return err
 	}

docker/docker_image_src.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/reference"
 	"github.com/projectatomic/skopeo/types"
 )
@@ -107,7 +107,7 @@ func (s *dockerImageSource) Delete() error {
 	// When retrieving the digest from a registry >= 2.3 use the following header:
 	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
 	headers := make(map[string][]string)
-	headers["Accept"] = []string{utils.DockerV2Schema2MIMEType}
+	headers["Accept"] = []string{manifest.DockerV2Schema2MIMEType}
 
 	getURL := fmt.Sprintf(manifestURL, s.ref.RemoteName(), s.tag)
 	get, err := s.c.makeRequest("GET", getURL, headers, nil)

docker/utils/fixtures_info_test.go

@@ -1,8 +0,0 @@
-package utils
-
-const (
-	// TestV2S2ManifestDigest is the Docker manifest digest of "v2s2.manifest.json"
-	TestV2S2ManifestDigest = "sha256:20bf21ed457b390829cdbeec8795a7bea1626991fda603e0d01b4e7f60427e55"
-	// TestV2S1ManifestDigest is the Docker manifest digest of "v2s1.manifest.json"
-	TestV2S1ManifestDigest = "sha256:077594da70fc17ec2c93cfa4e6ed1fcc26992851fb2c71861338aaf4aa9e41b1"
-)

image/image.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/projectatomic/skopeo/directory"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/types"
 )
 
@@ -47,7 +47,7 @@ func (i *genericImage) IntendedDockerReference() string {
 // Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need.
 func (i *genericImage) Manifest() ([]byte, error) {
 	if i.cachedManifest == nil {
-		m, _, err := i.src.GetManifest([]string{utils.DockerV2Schema1MIMEType})
+		m, _, err := i.src.GetManifest([]string{manifest.DockerV2Schema1MIMEType})
 		if err != nil {
 			return nil, err
 		}
@@ -116,7 +116,7 @@ func (i *genericImage) DockerTar() ([]byte, error) {
 }
 
 // will support v1 one day...
-type manifest interface {
+type genericManifest interface {
 	String() string
 	GetLayers() []string
 }
@@ -150,7 +150,7 @@ func sanitize(s string) string {
 	return strings.Replace(s, "/", "-", -1)
 }
 
-func (i *genericImage) getSchema1Manifest() (manifest, error) {
+func (i *genericImage) getSchema1Manifest() (genericManifest, error) {
 	manblob, err := i.Manifest()
 	if err != nil {
 		return nil, err

integration/copy_test.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/go-check/check"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 )
 
 func init() {
@@ -86,9 +86,9 @@ func (s *CopySuite) TestCopyStreaming(c *check.C) {
 	digests := []string{}
 	for _, dir := range []string{dir1, dir2} {
 		manifestPath := filepath.Join(dir, "manifest.json")
-		manifest, err := ioutil.ReadFile(manifestPath)
+		m, err := ioutil.ReadFile(manifestPath)
 		c.Assert(err, check.IsNil)
-		digest, err := utils.ManifestDigest(manifest)
+		digest, err := manifest.Digest(m)
 		c.Assert(err, check.IsNil)
 		digests = append(digests, digest)
 		err = os.Remove(manifestPath)

manifest/fixtures_info_test.go

@@ -0,0 +1,8 @@
+package manifest
+
+const (
+	// TestV2S2ManifestDigest is the Docker manifest digest of "v2s2.manifest.json"
+	TestDockerV2S2ManifestDigest = "sha256:20bf21ed457b390829cdbeec8795a7bea1626991fda603e0d01b4e7f60427e55"
+	// TestV2S1ManifestDigest is the Docker manifest digest of "v2s1.manifest.json"
+	TestDockerV2S1ManifestDigest = "sha256:077594da70fc17ec2c93cfa4e6ed1fcc26992851fb2c71861338aaf4aa9e41b1"
+)

manifest/manifest.go

@@ -1,4 +1,4 @@
-package utils
+package manifest
 
 import (
 	"crypto/sha256"
@@ -10,6 +10,7 @@ import (
 
 // FIXME: Should we just use docker/distribution and docker/docker implementations directly?
 
+// FIXME(runcom, mitr): should we havea mediatype pkg??
 const (
 	// DockerV2Schema1MIMEType MIME type represents Docker manifest schema 1
 	DockerV2Schema1MIMEType = "application/vnd.docker.distribution.manifest.v1+json"
@@ -17,12 +18,25 @@ const (
 	DockerV2Schema2MIMEType = "application/vnd.docker.distribution.manifest.v2+json"
 	// DockerV2ListMIMEType MIME type represents Docker manifest schema 2 list
 	DockerV2ListMIMEType = "application/vnd.docker.distribution.manifest.list.v2+json"
+
+	// OCIV1DescriptorMIMEType TODO
+	OCIV1DescriptorMIMEType = "application/vnd.oci.descriptor.v1+json"
+	// OCIV1ImageManifestMIMEType TODO
+	OCIV1ImageManifestMIMEType = "application/vnd.oci.image.manifest.v1+json"
+	// OCIV1ImageManifestListMIMEType TODO
+	OCIV1ImageManifestListMIMEType = "application/vnd.oci.image.manifest.list.v1+json"
+	// OCIV1ImageSerializationRootfsTarGzipMIMEType TODO)
+	OCIV1ImageSerializationRootfsTarGzipMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip"
+	// OCIV1ImageSerializationConfigMIMEType TODO
+	OCIV1ImageSerializationConfigMIMEType = "application/vnd.oci.image.serialization.config.v1+json"
+	// OCIV1ImageSerializationCombinedMIMEType TODO
+	OCIV1ImageSerializationCombinedMIMEType = "application/vnd.oci.image.serialization.combined.v1+json"
 )
 
-// GuessManifestMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized.
+// GuessMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized.
 // FIXME? We should, in general, prefer out-of-band MIME type instead of blindly parsing the manifest,
 // but we may not have such metadata available (e.g. when the manifest is a local file).
-func GuessManifestMIMEType(manifest []byte) string {
+func GuessMIMEType(manifest []byte) string {
 	// A subset of manifest fields; the rest is silently ignored by json.Unmarshal.
 	// Also docker/distribution/manifest.Versioned.
 	meta := struct {
@@ -34,9 +48,10 @@ func GuessManifestMIMEType(manifest []byte) string {
 	}
 
 	switch meta.MediaType {
-	case DockerV2Schema2MIMEType, DockerV2ListMIMEType: // A recognized type.
+	case DockerV2Schema2MIMEType, DockerV2ListMIMEType, OCIV1DescriptorMIMEType, OCIV1ImageManifestMIMEType, OCIV1ImageManifestListMIMEType, OCIV1ImageSerializationRootfsTarGzipMIMEType, OCIV1ImageSerializationConfigMIMEType, OCIV1ImageSerializationCombinedMIMEType: // A recognized type.
 		return meta.MediaType
 	}
+	// this is the only way the function can return DockerV2Schema1MIMEType, and recognizing that is essential for stripping the JWS signatures = computing the correct manifest digest.
 	switch meta.SchemaVersion {
 	case 1:
 		return DockerV2Schema1MIMEType
@@ -46,9 +61,9 @@ func GuessManifestMIMEType(manifest []byte) string {
 	return ""
 }
 
-// ManifestDigest returns the a digest of a docker manifest, with any necessary implied transformations like stripping v1s1 signatures.
+// Digest returns the a digest of a docker manifest, with any necessary implied transformations like stripping v1s1 signatures.
-func ManifestDigest(manifest []byte) (string, error) {
+func Digest(manifest []byte) (string, error) {
-	if GuessManifestMIMEType(manifest) == DockerV2Schema1MIMEType {
+	if GuessMIMEType(manifest) == DockerV2Schema1MIMEType {
 		sig, err := libtrust.ParsePrettySignature(manifest, "signatures")
 		if err != nil {
 			return "", err
@@ -65,13 +80,13 @@ func ManifestDigest(manifest []byte) (string, error) {
 	return "sha256:" + hex.EncodeToString(hash[:]), nil
 }
 
-// ManifestMatchesDigest returns true iff the manifest matches expectedDigest.
+// MatchesDigest returns true iff the manifest matches expectedDigest.
 // Error may be set if this returns false.
 // Note that this is not doing ConstantTimeCompare; by the time we get here, the cryptographic signature must already have been verified,
 // or we are not using a cryptographic channel and the attacker can modify the digest along with the manifest blob.
-func ManifestMatchesDigest(manifest []byte, expectedDigest string) (bool, error) {
+func MatchesDigest(manifest []byte, expectedDigest string) (bool, error) {
 	// This should eventually support various digest types.
-	actualDigest, err := ManifestDigest(manifest)
+	actualDigest, err := Digest(manifest)
 	if err != nil {
 		return false, err
 	}

manifest/manifest_test.go

@@ -1,4 +1,4 @@
-package utils
+package manifest
 
 import (
 	"crypto/sha256"
@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestGuessManifestMIMEType(t *testing.T) {
+func TestGuessMIMEType(t *testing.T) {
 	cases := []struct {
 		path     string
 		mimeType string
@@ -28,60 +28,60 @@ func TestGuessManifestMIMEType(t *testing.T) {
 	for _, c := range cases {
 		manifest, err := ioutil.ReadFile(filepath.Join("fixtures", c.path))
 		require.NoError(t, err)
-		mimeType := GuessManifestMIMEType(manifest)
+		mimeType := GuessMIMEType(manifest)
 		assert.Equal(t, c.mimeType, mimeType)
 	}
 }
 
-func TestManifestDigest(t *testing.T) {
+func TestDigest(t *testing.T) {
 	cases := []struct {
 		path   string
 		digest string
 	}{
-		{"v2s2.manifest.json", TestV2S2ManifestDigest},
+		{"v2s2.manifest.json", TestDockerV2S2ManifestDigest},
-		{"v2s1.manifest.json", TestV2S1ManifestDigest},
+		{"v2s1.manifest.json", TestDockerV2S1ManifestDigest},
 	}
 	for _, c := range cases {
 		manifest, err := ioutil.ReadFile(filepath.Join("fixtures", c.path))
 		require.NoError(t, err)
-		digest, err := ManifestDigest(manifest)
+		digest, err := Digest(manifest)
 		require.NoError(t, err)
 		assert.Equal(t, c.digest, digest)
 	}
 
 	manifest, err := ioutil.ReadFile("fixtures/v2s1-invalid-signatures.manifest.json")
 	require.NoError(t, err)
-	digest, err := ManifestDigest(manifest)
+	digest, err := Digest(manifest)
 	assert.Error(t, err)
 
-	digest, err = ManifestDigest([]byte{})
+	digest, err = Digest([]byte{})
 	require.NoError(t, err)
 	assert.Equal(t, "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", digest)
 }
 
-func TestManifestMatchesDigest(t *testing.T) {
+func TestMatchesDigest(t *testing.T) {
 	cases := []struct {
 		path   string
 		digest string
 		result bool
 	}{
 		// Success
-		{"v2s2.manifest.json", TestV2S2ManifestDigest, true},
+		{"v2s2.manifest.json", TestDockerV2S2ManifestDigest, true},
-		{"v2s1.manifest.json", TestV2S1ManifestDigest, true},
+		{"v2s1.manifest.json", TestDockerV2S1ManifestDigest, true},
 		// No match (switched s1/s2)
-		{"v2s2.manifest.json", TestV2S1ManifestDigest, false},
+		{"v2s2.manifest.json", TestDockerV2S1ManifestDigest, false},
-		{"v2s1.manifest.json", TestV2S2ManifestDigest, false},
+		{"v2s1.manifest.json", TestDockerV2S2ManifestDigest, false},
 		// Unrecognized algorithm
 		{"v2s2.manifest.json", "md5:2872f31c5c1f62a694fbd20c1e85257c", false},
 		// Mangled format
-		{"v2s2.manifest.json", TestV2S2ManifestDigest + "abc", false},
+		{"v2s2.manifest.json", TestDockerV2S2ManifestDigest + "abc", false},
-		{"v2s2.manifest.json", TestV2S2ManifestDigest[:20], false},
+		{"v2s2.manifest.json", TestDockerV2S2ManifestDigest[:20], false},
 		{"v2s2.manifest.json", "", false},
 	}
 	for _, c := range cases {
 		manifest, err := ioutil.ReadFile(filepath.Join("fixtures", c.path))
 		require.NoError(t, err)
-		res, err := ManifestMatchesDigest(manifest, c.digest)
+		res, err := MatchesDigest(manifest, c.digest)
 		require.NoError(t, err)
 		assert.Equal(t, c.result, res)
 	}
@@ -90,11 +90,11 @@ func TestManifestMatchesDigest(t *testing.T) {
 	require.NoError(t, err)
 	// Even a correct SHA256 hash is rejected if we can't strip the JSON signature.
 	hash := sha256.Sum256(manifest)
-	res, err := ManifestMatchesDigest(manifest, "sha256:"+hex.EncodeToString(hash[:]))
+	res, err := MatchesDigest(manifest, "sha256:"+hex.EncodeToString(hash[:]))
 	assert.False(t, res)
 	assert.Error(t, err)
 
-	res, err = ManifestMatchesDigest([]byte{}, "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	res, err = MatchesDigest([]byte{}, "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
 	assert.True(t, res)
 	assert.NoError(t, err)
 }

openshift/openshift.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/projectatomic/skopeo/docker"
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/types"
 	"github.com/projectatomic/skopeo/version"
 )
@@ -287,9 +287,9 @@ func (d *openshiftImageDestination) CanonicalDockerReference() (string, error) {
 	return d.client.canonicalDockerReference(), nil
 }
 
-func (d *openshiftImageDestination) PutManifest(manifest []byte) error {
+func (d *openshiftImageDestination) PutManifest(m []byte) error {
 	// Note: This does absolutely no kind/version checking or conversions.
-	manifestDigest, err := utils.ManifestDigest(manifest)
+	manifestDigest, err := manifest.Digest(m)
 	if err != nil {
 		return err
 	}
@@ -309,7 +309,7 @@ func (d *openshiftImageDestination) PutManifest(manifest []byte) error {
 				Name: manifestDigest,
 			},
 			DockerImageReference: dockerImageReference,
-			DockerImageManifest:  string(manifest),
+			DockerImageManifest:  string(m),
 		},
 		Tag: d.client.tag,
 	}
@@ -325,7 +325,7 @@ func (d *openshiftImageDestination) PutManifest(manifest []byte) error {
 		return err
 	}
 
-	return d.docker.PutManifest(manifest)
+	return d.docker.PutManifest(m)
 }
 
 func (d *openshiftImageDestination) PutBlob(digest string, stream io.Reader) error {

signature/docker.go

@@ -5,13 +5,13 @@ package signature
 import (
 	"fmt"
 
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 )
 
 // SignDockerManifest returns a signature for manifest as the specified dockerReference,
 // using mech and keyIdentity.
-func SignDockerManifest(manifest []byte, dockerReference string, mech SigningMechanism, keyIdentity string) ([]byte, error) {
+func SignDockerManifest(m []byte, dockerReference string, mech SigningMechanism, keyIdentity string) ([]byte, error) {
-	manifestDigest, err := utils.ManifestDigest(manifest)
+	manifestDigest, err := manifest.Digest(m)
 	if err != nil {
 		return nil, err
 	}
@@ -43,7 +43,7 @@ func VerifyDockerManifestSignature(unverifiedSignature, unverifiedManifest []byt
 			return nil
 		},
 		validateSignedDockerManifestDigest: func(signedDockerManifestDigest string) error {
-			matches, err := utils.ManifestMatchesDigest(unverifiedManifest, signedDockerManifestDigest)
+			matches, err := manifest.MatchesDigest(unverifiedManifest, signedDockerManifestDigest)
 			if err != nil {
 				return err
 			}

signature/policy_eval_signedby.go

@@ -9,7 +9,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/projectatomic/skopeo/docker/utils"
+	"github.com/projectatomic/skopeo/manifest"
 	"github.com/projectatomic/skopeo/types"
 )
 
@@ -76,11 +76,11 @@ func (pr *prSignedBy) isSignatureAuthorAccepted(image types.Image, sig []byte) (
 			return nil
 		},
 		validateSignedDockerManifestDigest: func(digest string) error {
-			manifest, err := image.Manifest()
+			m, err := image.Manifest()
 			if err != nil {
 				return err
 			}
-			digestMatches, err := utils.ManifestMatchesDigest(manifest, digest)
+			digestMatches, err := manifest.MatchesDigest(m, digest)
 			if err != nil {
 				return err
 			}