From 6942920ee8ecf1816365b5e8a081c3b66d1293e4 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 29 Jun 2016 23:42:04 +0200 Subject: [PATCH] add possibility to download to OCI image-layout - vendor containers/image c703326038d30c3422168dd9a1a5afaf51740331 - fix copy tests relying on v2s1 manifests Signed-off-by: Antonio Murdaca --- cmd/skopeo/copy.go | 13 +- cmd/skopeo/inspect.go | 2 +- cmd/skopeo/layers.go | 8 +- cmd/skopeo/utils.go | 7 +- integration/copy_test.go | 15 +- .../image/directory/directory_dest.go | 4 + .../containers/image/docker/docker_image.go | 2 +- .../image/docker/docker_image_dest.go | 11 +- .../image/docker/docker_image_src.go | 2 +- .../containers/image/image/image.go | 17 +- .../containers/image/manifest/manifest.go | 17 +- .../containers/image/oci/oci_dest.go | 183 ++++++++++++++++++ .../containers/image/openshift/openshift.go | 7 + .../containers/image/types/types.go | 3 + 14 files changed, 263 insertions(+), 28 deletions(-) create mode 100644 vendor/github.com/containers/image/oci/oci_dest.go diff --git a/cmd/skopeo/copy.go b/cmd/skopeo/copy.go index fbf18178..404f1dee 100644 --- a/cmd/skopeo/copy.go +++ b/cmd/skopeo/copy.go @@ -14,16 +14,17 @@ func copyHandler(context *cli.Context) error { return errors.New("Usage: copy source destination") } - rawSource, err := parseImageSource(context, context.Args()[0]) - if err != nil { - return fmt.Errorf("Error initializing %s: %v", context.Args()[0], err) - } - src := image.FromSource(rawSource) - dest, err := parseImageDestination(context, context.Args()[1]) if err != nil { return fmt.Errorf("Error initializing %s: %v", context.Args()[1], err) } + + rawSource, err := parseImageSource(context, context.Args()[0]) + if err != nil { + return fmt.Errorf("Error initializing %s: %v", context.Args()[0], err) + } + src := image.FromSource(rawSource, dest.SupportedManifestMIMETypes()) + signBy := context.String("sign-by") manifest, _, err := src.Manifest() diff --git a/cmd/skopeo/inspect.go b/cmd/skopeo/inspect.go index 49b40807..4e8376aa 100644 --- a/cmd/skopeo/inspect.go +++ b/cmd/skopeo/inspect.go @@ -13,7 +13,7 @@ import ( // inspectOutput is the output format of (skopeo inspect), primarily so that we can format it with a simple json.MarshalIndent. type inspectOutput struct { Name string `json:",omitempty"` - Tag string + Tag string `json:",omitempty"` Digest string RepoTags []string Created time.Time diff --git a/cmd/skopeo/layers.go b/cmd/skopeo/layers.go index 579b33f8..643b6ee8 100644 --- a/cmd/skopeo/layers.go +++ b/cmd/skopeo/layers.go @@ -6,6 +6,7 @@ import ( "github.com/containers/image/directory" "github.com/containers/image/image" + "github.com/containers/image/manifest" "github.com/urfave/cli" ) @@ -18,7 +19,12 @@ var layersCmd = cli.Command{ if err != nil { return err } - src := image.FromSource(rawSource) + src := image.FromSource(rawSource, []string{ + // TODO: skopeo layers only support these now + // eventually we'll remove this command altogether... + manifest.DockerV2Schema1SignedMIMEType, + manifest.DockerV2Schema1MIMEType, + }) blobDigests := c.Args().Tail() if len(blobDigests) == 0 { b, err := src.BlobDigests() diff --git a/cmd/skopeo/utils.go b/cmd/skopeo/utils.go index 86de067c..237036cf 100644 --- a/cmd/skopeo/utils.go +++ b/cmd/skopeo/utils.go @@ -8,6 +8,7 @@ import ( "github.com/containers/image/directory" "github.com/containers/image/docker" "github.com/containers/image/image" + "github.com/containers/image/oci" "github.com/containers/image/openshift" "github.com/containers/image/types" "github.com/urfave/cli" @@ -20,6 +21,8 @@ const ( dockerPrefix = "docker://" // directoryPrefix is the URL-like schema prefix used for local directories (for debugging) directoryPrefix = "dir:" + // ociPrefix is the URL-like schema prefix used for OCI images. + ociPrefix = "oci:" ) // ParseImage converts image URL-like string to an initialized handler for that image. @@ -36,7 +39,7 @@ func parseImage(c *cli.Context) (types.Image, error) { // case strings.HasPrefix(imgName, directoryPrefix): src := directory.NewDirImageSource(strings.TrimPrefix(imgName, directoryPrefix)) - return image.FromSource(src), nil + return image.FromSource(src, nil), nil } return nil, errors.New("no valid prefix provided") } @@ -71,6 +74,8 @@ func parseImageDestination(c *cli.Context, name string) (types.ImageDestination, return openshift.NewOpenshiftImageDestination(strings.TrimPrefix(name, atomicPrefix), certPath, tlsVerify) case strings.HasPrefix(name, directoryPrefix): return directory.NewDirImageDestination(strings.TrimPrefix(name, directoryPrefix)), nil + case strings.HasPrefix(name, ociPrefix): + return oci.NewOCIImageDestination(strings.TrimPrefix(name, ociPrefix)) } return nil, fmt.Errorf("Unrecognized image reference %s", name) } diff --git a/integration/copy_test.go b/integration/copy_test.go index c23d4398..f7a92dd5 100644 --- a/integration/copy_test.go +++ b/integration/copy_test.go @@ -55,7 +55,7 @@ func (s *CopySuite) TestCopySimple(c *check.C) { // FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection. // "pull": docker: → dir: - assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "dir:"+dir1) + assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "dir:"+dir1) // "push": dir: → atomic: assertSkopeoSucceeds(c, "", "--debug", "copy", "dir:"+dir1, "atomic:myns/unsigned:unsigned") // The result of pushing and pulling is an unmodified image. @@ -63,6 +63,15 @@ func (s *CopySuite) TestCopySimple(c *check.C) { out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2) c.Assert(out, check.Equals, "") + // docker v2s2 -> OCI image layout + // ociDest will be created by oci: if it doesn't exist + // so don't create it here to exercise auto-creation + ociDest := "busybox-latest" + defer os.RemoveAll(ociDest) + assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest) + _, err = os.Stat(ociDest) + c.Assert(err, check.IsNil) + // FIXME: Also check pushing to docker:// } @@ -77,9 +86,9 @@ func (s *CopySuite) TestCopyStreaming(c *check.C) { // FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection. // streaming: docker: → atomic: - assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://busybox:1-glibc", "atomic:myns/unsigned:streaming") + assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://estesp/busybox:amd64", "atomic:myns/unsigned:streaming") // Compare (copies of) the original and the copy: - assertSkopeoSucceeds(c, "", "copy", "docker://busybox:1-glibc", "dir:"+dir1) + assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:amd64", "dir:"+dir1) assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:streaming", "dir:"+dir2) // The manifests will have different JWS signatures; so, compare the manifests by digests, which // strips the signatures, and remove them, comparing the rest file by file. diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index aba18c60..09547cb7 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -22,6 +22,10 @@ func (d *dirImageDestination) CanonicalDockerReference() (string, error) { return "", fmt.Errorf("Can not determine canonical Docker reference for a local directory") } +func (d *dirImageDestination) SupportedManifestMIMETypes() []string { + return nil +} + func (d *dirImageDestination) PutManifest(manifest []byte) error { return ioutil.WriteFile(manifestPath(d.dir), manifest, 0644) } diff --git a/vendor/github.com/containers/image/docker/docker_image.go b/vendor/github.com/containers/image/docker/docker_image.go index a4ce9743..dd7bad0a 100644 --- a/vendor/github.com/containers/image/docker/docker_image.go +++ b/vendor/github.com/containers/image/docker/docker_image.go @@ -23,7 +23,7 @@ func NewDockerImage(img, certPath string, tlsVerify bool) (types.Image, error) { if err != nil { return nil, err } - return &Image{Image: image.FromSource(s), src: s}, nil + return &Image{Image: image.FromSource(s, nil), src: s}, nil } // SourceRefFullName returns a fully expanded name for the repository this image is in. diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 3b99614d..67987f40 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -8,8 +8,8 @@ import ( "net/http" "github.com/Sirupsen/logrus" - "github.com/containers/image/manifest" "github.com/containers/image/docker/reference" + "github.com/containers/image/manifest" "github.com/containers/image/types" ) @@ -36,6 +36,15 @@ func NewDockerImageDestination(img, certPath string, tlsVerify bool) (types.Imag }, nil } +func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { + return []string{ + // TODO(runcom): we'll add OCI as part of another PR here + manifest.DockerV2Schema2MIMEType, + manifest.DockerV2Schema1SignedMIMEType, + manifest.DockerV2Schema1MIMEType, + } +} + func (d *dockerImageDestination) CanonicalDockerReference() (string, error) { return fmt.Sprintf("%s:%s", d.ref.Name(), d.tag), nil } diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 8371f418..b4152a00 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -95,7 +95,7 @@ func (s *dockerImageSource) GetManifest(mimetypes []string) ([]byte, string, err func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) { url := fmt.Sprintf(blobsURL, s.ref.RemoteName(), digest) - logrus.Infof("Downloading %s", url) + logrus.Debugf("Downloading %s", url) res, err := s.c.makeRequest("GET", url, nil, nil) if err != nil { return nil, 0, err diff --git a/vendor/github.com/containers/image/image/image.go b/vendor/github.com/containers/image/image/image.go index 5850c40b..ac84949d 100644 --- a/vendor/github.com/containers/image/image/image.go +++ b/vendor/github.com/containers/image/image/image.go @@ -33,12 +33,21 @@ type genericImage struct { // this field is valid only if cachedManifest is not nil cachedManifestMIMEType string // private cache for Signatures(); nil if not yet known. - cachedSignatures [][]byte + cachedSignatures [][]byte + requestedManifestMIMETypes []string } // FromSource returns a types.Image implementation for source. -func FromSource(src types.ImageSource) types.Image { - return &genericImage{src: src} +func FromSource(src types.ImageSource, requestedManifestMIMETypes []string) types.Image { + if len(requestedManifestMIMETypes) == 0 { + requestedManifestMIMETypes = []string{ + manifest.OCIV1ImageManifestMIMEType, + manifest.DockerV2Schema2MIMEType, + manifest.DockerV2Schema1SignedMIMEType, + manifest.DockerV2Schema1MIMEType, + } + } + return &genericImage{src: src, requestedManifestMIMETypes: requestedManifestMIMETypes} } // IntendedDockerReference returns the full, unambiguous, Docker reference for this image, _as specified by the user_ @@ -52,7 +61,7 @@ func (i *genericImage) IntendedDockerReference() string { // NOTE: It is essential for signature verification that Manifest returns the manifest from which BlobDigests is computed. func (i *genericImage) Manifest() ([]byte, string, error) { if i.cachedManifest == nil { - m, mt, err := i.src.GetManifest([]string{manifest.DockerV2Schema1SignedMIMEType, manifest.DockerV2Schema1MIMEType}) + m, mt, err := i.src.GetManifest(i.requestedManifestMIMETypes) if err != nil { return nil, "", err } diff --git a/vendor/github.com/containers/image/manifest/manifest.go b/vendor/github.com/containers/image/manifest/manifest.go index 1b3a76ca..9345b556 100644 --- a/vendor/github.com/containers/image/manifest/manifest.go +++ b/vendor/github.com/containers/image/manifest/manifest.go @@ -20,18 +20,17 @@ const ( DockerV2Schema2MIMEType = "application/vnd.docker.distribution.manifest.v2+json" // DockerV2ListMIMEType MIME type represents Docker manifest schema 2 list DockerV2ListMIMEType = "application/vnd.docker.distribution.manifest.list.v2+json" - // OCIV1DescriptorMIMEType TODO + + // OCIV1DescriptorMIMEType specifies the mediaType for a content descriptor. OCIV1DescriptorMIMEType = "application/vnd.oci.descriptor.v1+json" - // OCIV1ImageManifestMIMEType TODO + // OCIV1ImageManifestMIMEType specifies the mediaType for an image manifest. OCIV1ImageManifestMIMEType = "application/vnd.oci.image.manifest.v1+json" - // OCIV1ImageManifestListMIMEType TODO + // OCIV1ImageManifestListMIMEType specifies the mediaType for an image manifest list. OCIV1ImageManifestListMIMEType = "application/vnd.oci.image.manifest.list.v1+json" - // OCIV1ImageSerializationRootfsTarGzipMIMEType TODO) - OCIV1ImageSerializationRootfsTarGzipMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip" - // OCIV1ImageSerializationConfigMIMEType TODO + // OCIV1ImageSerializationMIMEType is the mediaType used for layers referenced by the manifest. + OCIV1ImageSerializationMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip" + // OCIV1ImageSerializationConfigMIMEType specifies the mediaType for the image configuration. OCIV1ImageSerializationConfigMIMEType = "application/vnd.oci.image.serialization.config.v1+json" - // OCIV1ImageSerializationCombinedMIMEType TODO - OCIV1ImageSerializationCombinedMIMEType = "application/vnd.oci.image.serialization.combined.v1+json" ) // GuessMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized. @@ -50,7 +49,7 @@ func GuessMIMEType(manifest []byte) string { } switch meta.MediaType { - case DockerV2Schema2MIMEType, DockerV2ListMIMEType, OCIV1DescriptorMIMEType, OCIV1ImageManifestMIMEType, OCIV1ImageManifestListMIMEType, OCIV1ImageSerializationRootfsTarGzipMIMEType, OCIV1ImageSerializationConfigMIMEType, OCIV1ImageSerializationCombinedMIMEType: // A recognized type. + case DockerV2Schema2MIMEType, DockerV2ListMIMEType, OCIV1DescriptorMIMEType, OCIV1ImageManifestMIMEType, OCIV1ImageManifestListMIMEType: // A recognized type. return meta.MediaType } // this is the only way the function can return DockerV2Schema1MIMEType, and recognizing that is essential for stripping the JWS signatures = computing the correct manifest digest. diff --git a/vendor/github.com/containers/image/oci/oci_dest.go b/vendor/github.com/containers/image/oci/oci_dest.go new file mode 100644 index 00000000..392ae4d4 --- /dev/null +++ b/vendor/github.com/containers/image/oci/oci_dest.go @@ -0,0 +1,183 @@ +package oci + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/containers/image/manifest" + "github.com/containers/image/types" +) + +type ociManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + Annotations map[string]string `json:"annotations"` +} + +type descriptor struct { + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + Size int64 `json:"size"` +} + +type ociImageDestination struct { + dir string + tag string +} + +var refRegexp = regexp.MustCompile(`^([A-Za-z0-9._-]+)+$`) + +// NewOCIImageDestination returns an ImageDestination for writing to an existing directory. +func NewOCIImageDestination(dest string) (types.ImageDestination, error) { + dir := dest + sep := strings.LastIndex(dest, ":") + tag := "latest" + if sep != -1 { + dir = dest[:sep] + tag = dest[sep+1:] + if !refRegexp.MatchString(tag) { + return nil, fmt.Errorf("Invalid reference %s", tag) + } + } + return &ociImageDestination{ + dir: dir, + tag: tag, + }, nil +} + +func (d *ociImageDestination) CanonicalDockerReference() (string, error) { + return "", fmt.Errorf("Can not determine canonical Docker reference for an OCI image") +} + +func createManifest(m []byte) ([]byte, string, error) { + om := ociManifest{} + mt := manifest.GuessMIMEType(m) + switch mt { + case manifest.DockerV2Schema1MIMEType: + // There a simple reason about not yet implementing this. + // OCI image-spec assure about backward compatibility with docker v2s2 but not v2s1 + // generating a v2s2 is a migration docker does when upgrading to 1.10.3 + // and I don't think we should bother about this now (I don't want to have migration code here in skopeo) + return nil, "", fmt.Errorf("can't create OCI manifest from Docker V2 schema 1 manifest") + case manifest.DockerV2Schema2MIMEType: + if err := json.Unmarshal(m, &om); err != nil { + return nil, "", err + } + om.MediaType = manifest.OCIV1ImageManifestMIMEType + for i := range om.Layers { + om.Layers[i].MediaType = manifest.OCIV1ImageSerializationMIMEType + } + om.Config.MediaType = manifest.OCIV1ImageSerializationConfigMIMEType + b, err := json.Marshal(om) + if err != nil { + return nil, "", err + } + return b, om.MediaType, nil + case manifest.DockerV2ListMIMEType: + return nil, "", fmt.Errorf("can't create OCI manifest from Docker V2 schema 2 manifest list") + case manifest.OCIV1ImageManifestListMIMEType: + return nil, "", fmt.Errorf("can't create OCI manifest from OCI manifest list") + case manifest.OCIV1ImageManifestMIMEType: + return m, om.MediaType, nil + } + return nil, "", fmt.Errorf("Unrecognized manifest media type") +} + +func (d *ociImageDestination) PutManifest(m []byte) error { + if err := d.ensureParentDirectoryExists("refs"); err != nil { + return err + } + // TODO(mitr, runcom): this breaks signatures entirely since at this point we're creating a new manifest + // and signatures don't apply anymore. Will fix. + ociMan, mt, err := createManifest(m) + if err != nil { + return err + } + digest, err := manifest.Digest(ociMan) + if err != nil { + return err + } + desc := descriptor{} + desc.Digest = digest + // TODO(runcom): beaware and add support for OCI manifest list + desc.MediaType = mt + desc.Size = int64(len(ociMan)) + data, err := json.Marshal(desc) + if err != nil { + return err + } + + if err := ioutil.WriteFile(blobPath(d.dir, digest), ociMan, 0644); err != nil { + return err + } + // TODO(runcom): ugly here? + if err := ioutil.WriteFile(ociLayoutPath(d.dir), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0644); err != nil { + return err + } + return ioutil.WriteFile(descriptorPath(d.dir, d.tag), data, 0644) +} + +func (d *ociImageDestination) PutBlob(digest string, stream io.Reader) error { + if err := d.ensureParentDirectoryExists("blobs"); err != nil { + return err + } + blob, err := os.Create(blobPath(d.dir, digest)) + if err != nil { + return err + } + defer blob.Close() + if _, err := io.Copy(blob, stream); err != nil { + return err + } + if err := blob.Sync(); err != nil { + return err + } + return nil +} + +func (d *ociImageDestination) ensureParentDirectoryExists(parent string) error { + path := filepath.Join(d.dir, parent) + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + } + return nil +} + +func (d *ociImageDestination) SupportedManifestMIMETypes() []string { + return []string{ + manifest.OCIV1ImageManifestMIMEType, + manifest.DockerV2Schema2MIMEType, + } +} + +func (d *ociImageDestination) PutSignatures(signatures [][]byte) error { + if len(signatures) != 0 { + return fmt.Errorf("Pushing signatures for OCI images is not supported") + } + return nil +} + +// ociLayoutPathPath returns a path for the oci-layout within a directory using OCI conventions. +func ociLayoutPath(dir string) string { + return filepath.Join(dir, "oci-layout") +} + +// blobPath returns a path for a blob within a directory using OCI image-layout conventions. +func blobPath(dir string, digest string) string { + return filepath.Join(dir, "blobs", strings.Replace(digest, ":", "-", -1)) +} + +// descriptorPath returns a path for the manifest within a directory using OCI conventions. +func descriptorPath(dir string, digest string) string { + return filepath.Join(dir, "refs", digest) +} diff --git a/vendor/github.com/containers/image/openshift/openshift.go b/vendor/github.com/containers/image/openshift/openshift.go index 632bd1c5..d5579d3c 100644 --- a/vendor/github.com/containers/image/openshift/openshift.go +++ b/vendor/github.com/containers/image/openshift/openshift.go @@ -283,6 +283,13 @@ func NewOpenshiftImageDestination(imageName, certPath string, tlsVerify bool) (t }, nil } +func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { + return []string{ + manifest.DockerV2Schema1SignedMIMEType, + manifest.DockerV2Schema1MIMEType, + } +} + func (d *openshiftImageDestination) CanonicalDockerReference() (string, error) { return d.client.canonicalDockerReference(), nil } diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 212748bc..0bb94d21 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -34,6 +34,9 @@ type ImageDestination interface { // Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. PutBlob(digest string, stream io.Reader) error PutSignatures(signatures [][]byte) error + // SupportedManifestMIMETypes tells which manifest mime types the destination supports + // If an empty slice or nil it's returned, then any mime type can be tried to upload + SupportedManifestMIMETypes() []string } // Image is the primary API for inspecting properties of images.