diff --git a/cmd/skopeo/copy.go b/cmd/skopeo/copy.go index 725f74c8..8496620a 100644 --- a/cmd/skopeo/copy.go +++ b/cmd/skopeo/copy.go @@ -105,5 +105,10 @@ var copyCmd = cli.Command{ Name: "dest-tls-verify", Usage: "require HTTPS and verify certificates when talking to the docker destination registry (defaults to true)", }, + cli.StringFlag{ + Name: "dest-ostree-tmp-dir", + Value: "", + Usage: "`DIRECTORY` to use for OSTree temporary files", + }, }, } diff --git a/cmd/skopeo/utils.go b/cmd/skopeo/utils.go index cd4d8fe5..2841977c 100644 --- a/cmd/skopeo/utils.go +++ b/cmd/skopeo/utils.go @@ -16,6 +16,7 @@ func contextFromGlobalOptions(c *cli.Context, flagPrefix string) (*types.SystemC // DEPRECATED: keep this here for backward compatibility, but override // them if per subcommand flags are provided (see below). DockerInsecureSkipTLSVerify: !c.GlobalBoolT("tls-verify"), + OSTreeTmpDirPath: c.String(flagPrefix + "ostree-tmp-dir"), } if c.IsSet(flagPrefix + "tls-verify") { ctx.DockerInsecureSkipTLSVerify = !c.BoolT(flagPrefix + "tls-verify") diff --git a/completions/bash/skopeo b/completions/bash/skopeo index 6936910a..b21644c0 100644 --- a/completions/bash/skopeo +++ b/completions/bash/skopeo @@ -27,6 +27,7 @@ _skopeo_copy() { --src-tls-verify --dest-creds --dcreds --dest-cert-dir + --dest-ostree-tmp-dir --dest-tls-verify " local boolean_options=" diff --git a/docs/skopeo.1.md b/docs/skopeo.1.md index 2a2feec7..ea89fde5 100644 --- a/docs/skopeo.1.md +++ b/docs/skopeo.1.md @@ -74,6 +74,8 @@ Uses the system's trust policy to validate images, rejects images not trusted by **--dest-cert-dir** _path_ Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the destination registry + **--dest-ostree-tmp-dir** _path_ Directory to use for OSTree temporary files. + **--dest-tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to docker destination registry (defaults to true) Existing signatures, if any, are preserved as well. diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go new file mode 100644 index 00000000..2bdf7ba6 --- /dev/null +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -0,0 +1,248 @@ +package ostree + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/containers/image/manifest" + "github.com/containers/image/types" + "github.com/containers/storage/pkg/archive" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type blobToImport struct { + Size int64 + Digest digest.Digest + BlobPath string +} + +type descriptor struct { + Size int64 `json:"size"` + Digest digest.Digest `json:"digest"` +} + +type manifestSchema struct { + ConfigDescriptor descriptor `json:"config"` + LayersDescriptors []descriptor `json:"layers"` +} + +type ostreeImageDestination struct { + ref ostreeReference + manifest string + schema manifestSchema + tmpDirPath string + blobs map[string]*blobToImport +} + +// newImageDestination returns an ImageDestination for writing to an existing ostree. +func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDestination, error) { + tmpDirPath = filepath.Join(tmpDirPath, ref.branchName) + if err := ensureDirectoryExists(tmpDirPath); err != nil { + return nil, err + } + return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}}, nil +} + +// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, +// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. +func (d *ostreeImageDestination) Reference() types.ImageReference { + return d.ref +} + +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *ostreeImageDestination) Close() error { + return os.RemoveAll(d.tmpDirPath) +} + +func (d *ostreeImageDestination) SupportedManifestMIMETypes() []string { + return []string{ + manifest.DockerV2Schema2MediaType, + } +} + +// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. +// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. +func (d *ostreeImageDestination) SupportsSignatures() error { + return nil +} + +// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination. +func (d *ostreeImageDestination) ShouldCompressLayers() bool { + return false +} + +// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually +// uploaded to the image destination, true otherwise. +func (d *ostreeImageDestination) AcceptsForeignLayerURLs() bool { + return false +} + +func (d *ostreeImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) { + tmpDir, err := ioutil.TempDir(d.tmpDirPath, "blob") + if err != nil { + return types.BlobInfo{}, err + } + + blobPath := filepath.Join(tmpDir, "content") + blobFile, err := os.Create(blobPath) + if err != nil { + return types.BlobInfo{}, err + } + defer blobFile.Close() + + digester := digest.Canonical.Digester() + tee := io.TeeReader(stream, digester.Hash()) + + size, err := io.Copy(blobFile, tee) + if err != nil { + return types.BlobInfo{}, err + } + computedDigest := digester.Digest() + if inputInfo.Size != -1 && size != inputInfo.Size { + return types.BlobInfo{}, errors.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, inputInfo.Size, size) + } + if err := blobFile.Sync(); err != nil { + return types.BlobInfo{}, err + } + + hash := computedDigest.Hex() + d.blobs[hash] = &blobToImport{Size: size, Digest: computedDigest, BlobPath: blobPath} + return types.BlobInfo{Digest: computedDigest, Size: size}, nil +} + +func (d *ostreeImageDestination) importBlob(blob *blobToImport) error { + ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex()) + destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root") + if err := ensureDirectoryExists(destinationPath); err != nil { + return err + } + defer func() { + os.Remove(blob.BlobPath) + os.RemoveAll(destinationPath) + }() + + err := archive.UntarPath(blob.BlobPath, destinationPath) + if err != nil { + return err + } + return exec.Command("ostree", "commit", + "--repo", d.ref.repo, + fmt.Sprintf("--add-metadata-string=docker.size=%d", blob.Size), + "--branch", ostreeBranch, + fmt.Sprintf("--tree=dir=%s", destinationPath)).Run() +} + +func (d *ostreeImageDestination) importConfig(blob *blobToImport) error { + ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex()) + + return exec.Command("ostree", "commit", + "--repo", d.ref.repo, + fmt.Sprintf("--add-metadata-string=docker.size=%d", blob.Size), + "--branch", ostreeBranch, filepath.Dir(blob.BlobPath)).Run() +} + +func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) { + branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex()) + output, err := exec.Command("ostree", "show", "--repo", d.ref.repo, "--print-metadata-key=docker.size", branch).CombinedOutput() + if err != nil { + if bytes.Index(output, []byte("not found")) >= 0 || bytes.Index(output, []byte("No such")) >= 0 { + return false, -1, nil + } + return false, -1, err + } + size, err := strconv.ParseInt(strings.Trim(string(output), "'\n"), 10, 64) + if err != nil { + return false, -1, err + } + + return true, size, nil +} + +func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) { + return info, nil +} + +func (d *ostreeImageDestination) PutManifest(manifest []byte) error { + d.manifest = string(manifest) + + if err := json.Unmarshal(manifest, &d.schema); err != nil { + return err + } + + manifestPath := filepath.Join(d.tmpDirPath, d.ref.manifestPath()) + if err := ensureParentDirectoryExists(manifestPath); err != nil { + return err + } + + return ioutil.WriteFile(manifestPath, manifest, 0644) +} + +func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error { + path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0)) + if err := ensureParentDirectoryExists(path); err != nil { + return err + } + + for i, sig := range signatures { + signaturePath := filepath.Join(d.tmpDirPath, d.ref.signaturePath(i)) + if err := ioutil.WriteFile(signaturePath, sig, 0644); err != nil { + return err + } + } + return nil +} + +func (d *ostreeImageDestination) Commit() error { + for _, layer := range d.schema.LayersDescriptors { + hash := layer.Digest.Hex() + blob := d.blobs[hash] + // if the blob is not present in d.blobs then it is already stored in OSTree, + // and we don't need to import it. + if blob == nil { + continue + } + err := d.importBlob(blob) + if err != nil { + return err + } + } + + hash := d.schema.ConfigDescriptor.Digest.Hex() + blob := d.blobs[hash] + if blob != nil { + err := d.importConfig(blob) + if err != nil { + return err + } + } + + manifestPath := filepath.Join(d.tmpDirPath, "manifest") + err := exec.Command("ostree", "commit", + "--repo", d.ref.repo, + fmt.Sprintf("--add-metadata-string=docker.manifest=%s", string(d.manifest)), + fmt.Sprintf("--branch=ociimage/%s", d.ref.branchName), + manifestPath).Run() + return err +} + +func ensureDirectoryExists(path string) error { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + } + return nil +} + +func ensureParentDirectoryExists(path string) error { + return ensureDirectoryExists(filepath.Dir(path)) +} diff --git a/vendor/github.com/containers/image/ostree/ostree_transport.go b/vendor/github.com/containers/image/ostree/ostree_transport.go new file mode 100644 index 00000000..f165b13f --- /dev/null +++ b/vendor/github.com/containers/image/ostree/ostree_transport.go @@ -0,0 +1,235 @@ +package ostree + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" + + "github.com/containers/image/directory/explicitfilepath" + "github.com/containers/image/docker/reference" + "github.com/containers/image/transports" + "github.com/containers/image/types" +) + +const defaultOSTreeRepo = "/ostree/repo" + +// Transport is an ImageTransport for ostree paths. +var Transport = ostreeTransport{} + +type ostreeTransport struct{} + +func (t ostreeTransport) Name() string { + return "ostree" +} + +func init() { + transports.Register(Transport) +} + +// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys +// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). +// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. +// scope passed to this function will not be "", that value is always allowed. +func (t ostreeTransport) ValidatePolicyConfigurationScope(scope string) error { + sep := strings.Index(scope, ":") + if sep < 0 { + return errors.Errorf("Invalid ostree: scope %s: Must include a repo", scope) + } + repo := scope[:sep] + + if !strings.HasPrefix(repo, "/") { + return errors.Errorf("Invalid ostree: scope %s: repository must be an absolute path", scope) + } + cleaned := filepath.Clean(repo) + if cleaned != repo { + return errors.Errorf(`Invalid ostree: scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) + } + + // FIXME? In the namespaces within a repo, + // we could be verifying the various character set and length restrictions + // from docker/distribution/reference.regexp.go, but other than that there + // are few semantically invalid strings. + return nil +} + +// ostreeReference is an ImageReference for ostree paths. +type ostreeReference struct { + image string + branchName string + repo string +} + +func (t ostreeTransport) ParseReference(ref string) (types.ImageReference, error) { + var repo = "" + var image = "" + s := strings.SplitN(ref, "@/", 2) + if len(s) == 1 { + image, repo = s[0], defaultOSTreeRepo + } else { + image, repo = s[0], "/"+s[1] + } + + return NewReference(image, repo) +} + +// NewReference returns an OSTree reference for a specified repo and image. +func NewReference(image string, repo string) (types.ImageReference, error) { + // image is not _really_ in a containers/image/docker/reference format; + // as far as the libOSTree ociimage/* namespace is concerned, it is more or + // less an arbitrary string with an implied tag. + // We use the reference.* parsers basically for the default tag name in + // reference.TagNameOnly, and incidentally for some character set and length + // restrictions. + var ostreeImage reference.Named + s := strings.SplitN(image, ":", 2) + + named, err := reference.WithName(s[0]) + if err != nil { + return nil, err + } + + if len(s) == 1 { + ostreeImage = reference.TagNameOnly(named) + } else { + ostreeImage, err = reference.WithTag(named, s[1]) + if err != nil { + return nil, err + } + } + + resolved, err := explicitfilepath.ResolvePathToFullyExplicit(repo) + if err != nil { + // With os.IsNotExist(err), the parent directory of repo is also not existent; + // that should ordinarily not happen, but it would be a bit weird to reject + // references which do not specify a repo just because the implicit defaultOSTreeRepo + // does not exist. + if os.IsNotExist(err) && repo == defaultOSTreeRepo { + resolved = repo + } else { + return nil, err + } + } + // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces + // from being ambiguous with values of PolicyConfigurationIdentity. + if strings.Contains(resolved, ":") { + return nil, errors.Errorf("Invalid OSTreeCI reference %s@%s: path %s contains a colon", image, repo, resolved) + } + + return ostreeReference{ + image: ostreeImage.String(), + branchName: encodeOStreeRef(ostreeImage.String()), + repo: resolved, + }, nil +} + +func (ref ostreeReference) Transport() types.ImageTransport { + return Transport +} + +// StringWithinTransport returns a string representation of the reference, which MUST be such that +// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. +// NOTE: The returned string is not promised to be equal to the original input to ParseReference; +// e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa. +// WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix. +func (ref ostreeReference) StringWithinTransport() string { + return fmt.Sprintf("%s@%s", ref.image, ref.repo) +} + +// DockerReference returns a Docker reference associated with this reference +// (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent, +// not e.g. after redirect or alias processing), or nil if unknown/not applicable. +func (ref ostreeReference) DockerReference() reference.Named { + return nil +} + +func (ref ostreeReference) PolicyConfigurationIdentity() string { + return fmt.Sprintf("%s:%s", ref.repo, ref.image) +} + +// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search +// for if explicit configuration for PolicyConfigurationIdentity() is not set. The list will be processed +// in order, terminating on first match, and an implicit "" is always checked at the end. +// It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(), +// and each following element to be a prefix of the element preceding it. +func (ref ostreeReference) PolicyConfigurationNamespaces() []string { + s := strings.SplitN(ref.image, ":", 2) + if len(s) != 2 { // Coverage: Should never happen, NewReference above ensures ref.image has a :tag. + panic(fmt.Sprintf("Internal inconsistency: ref.image value %q does not have a :tag", ref.image)) + } + name := s[0] + res := []string{} + for { + res = append(res, fmt.Sprintf("%s:%s", ref.repo, name)) + + lastSlash := strings.LastIndex(name, "/") + if lastSlash == -1 { + break + } + name = name[:lastSlash] + } + return res +} + +// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned Image. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +func (ref ostreeReference) NewImage(ctx *types.SystemContext) (types.Image, error) { + return nil, errors.New("Reading ostree: images is currently not supported") +} + +// NewImageSource returns a types.ImageSource for this reference, +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. +// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. +func (ref ostreeReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) { + return nil, errors.New("Reading ostree: images is currently not supported") +} + +// NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. +func (ref ostreeReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { + var tmpDir string + if ctx == nil || ctx.OSTreeTmpDirPath == "" { + tmpDir = os.TempDir() + } else { + tmpDir = ctx.OSTreeTmpDirPath + } + return newImageDestination(ref, tmpDir) +} + +// DeleteImage deletes the named image from the registry, if supported. +func (ref ostreeReference) DeleteImage(ctx *types.SystemContext) error { + return errors.Errorf("Deleting images not implemented for ostree: images") +} + +var ostreeRefRegexp = regexp.MustCompile(`^[A-Za-z0-9.-]$`) + +func encodeOStreeRef(in string) string { + var buffer bytes.Buffer + for i := range in { + sub := in[i : i+1] + if ostreeRefRegexp.MatchString(sub) { + buffer.WriteString(sub) + } else { + buffer.WriteString(fmt.Sprintf("_%02X", sub[0])) + } + + } + return buffer.String() +} + +// manifestPath returns a path for the manifest within a ostree using our conventions. +func (ref ostreeReference) manifestPath() string { + return filepath.Join("manifest", "manifest.json") +} + +// signaturePath returns a path for a signature within a ostree using our conventions. +func (ref ostreeReference) signaturePath(index int) string { + return filepath.Join("manifest", fmt.Sprintf("signature-%d", index+1)) +} diff --git a/vendor/github.com/containers/image/transports/alltransports/alltransports.go b/vendor/github.com/containers/image/transports/alltransports/alltransports.go index 1fdc22f8..dc70fadd 100644 --- a/vendor/github.com/containers/image/transports/alltransports/alltransports.go +++ b/vendor/github.com/containers/image/transports/alltransports/alltransports.go @@ -12,6 +12,7 @@ import ( _ "github.com/containers/image/docker/daemon" _ "github.com/containers/image/oci/layout" _ "github.com/containers/image/openshift" + _ "github.com/containers/image/ostree" _ "github.com/containers/image/storage" "github.com/containers/image/transports" "github.com/containers/image/types" diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index a32bf5a9..3be547cc 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -299,6 +299,8 @@ type SystemContext struct { // Note that this field is used mainly to integrate containers/image into projectatomic/docker // in order to not break any existing docker's integration tests. DockerDisableV1Ping bool + // Directory to use for OSTree temporary files + OSTreeTmpDirPath string } // ProgressProperties is used to pass information from the copy code to a monitor which