mirror of
https://github.com/containers/skopeo.git
synced 2025-05-10 17:05:26 +00:00
> go get github.com/containers/image/v5@main > make vendor Signed-off-by: Miloslav Trmač <mitr@redhat.com>
354 lines
16 KiB
Go
354 lines
16 KiB
Go
package copy
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/containers/image/v5/docker/reference"
|
|
internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
|
|
"github.com/containers/image/v5/internal/image"
|
|
"github.com/containers/image/v5/internal/imagedestination"
|
|
"github.com/containers/image/v5/internal/imagesource"
|
|
internalManifest "github.com/containers/image/v5/internal/manifest"
|
|
"github.com/containers/image/v5/internal/private"
|
|
"github.com/containers/image/v5/manifest"
|
|
"github.com/containers/image/v5/pkg/blobinfocache"
|
|
"github.com/containers/image/v5/signature"
|
|
"github.com/containers/image/v5/signature/signer"
|
|
"github.com/containers/image/v5/transports"
|
|
"github.com/containers/image/v5/types"
|
|
encconfig "github.com/containers/ocicrypt/config"
|
|
digest "github.com/opencontainers/go-digest"
|
|
"github.com/sirupsen/logrus"
|
|
"golang.org/x/exp/slices"
|
|
"golang.org/x/sync/semaphore"
|
|
"golang.org/x/term"
|
|
)
|
|
|
|
var (
|
|
// ErrDecryptParamsMissing is returned if there is missing decryption parameters
|
|
ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present")
|
|
|
|
// maxParallelDownloads is used to limit the maximum number of parallel
|
|
// downloads. Let's follow Firefox by limiting it to 6.
|
|
maxParallelDownloads = uint(6)
|
|
)
|
|
|
|
const (
|
|
// CopySystemImage is the default value which, when set in
|
|
// Options.ImageListSelection, indicates that the caller expects only one
|
|
// image to be copied, so if the source reference refers to a list of
|
|
// images, one that matches the current system will be selected.
|
|
CopySystemImage ImageListSelection = iota
|
|
// CopyAllImages is a value which, when set in Options.ImageListSelection,
|
|
// indicates that the caller expects to copy multiple images, and if
|
|
// the source reference refers to a list, that the list and every image
|
|
// to which it refers will be copied. If the source reference refers
|
|
// to a list, the target reference can not accept lists, an error
|
|
// should be returned.
|
|
CopyAllImages
|
|
// CopySpecificImages is a value which, when set in
|
|
// Options.ImageListSelection, indicates that the caller expects the
|
|
// source reference to be either a single image or a list of images,
|
|
// and if the source reference is a list, wants only specific instances
|
|
// from it copied (or none of them, if the list of instances to copy is
|
|
// empty), along with the list itself. If the target reference can
|
|
// only accept one image (i.e., it cannot accept lists), an error
|
|
// should be returned.
|
|
CopySpecificImages
|
|
)
|
|
|
|
// ImageListSelection is one of CopySystemImage, CopyAllImages, or
|
|
// CopySpecificImages, to control whether, when the source reference is a list,
|
|
// copy.Image() copies only an image which matches the current runtime
|
|
// environment, or all images which match the supplied reference, or only
|
|
// specific images from the source reference.
|
|
type ImageListSelection int
|
|
|
|
// Options allows supplying non-default configuration modifying the behavior of CopyImage.
|
|
type Options struct {
|
|
RemoveSignatures bool // Remove any pre-existing signatures. Signers and SignBy… will still add a new signature.
|
|
// Signers to use to add signatures during the copy.
|
|
// Callers are still responsible for closing these Signer objects; they can be reused for multiple copy.Image operations in a row.
|
|
Signers []*signer.Signer
|
|
SignBy string // If non-empty, asks for a signature to be added during the copy, and specifies a key ID, as accepted by signature.NewGPGSigningMechanism().SignDockerManifest(),
|
|
SignPassphrase string // Passphrase to use when signing with the key ID from `SignBy`.
|
|
SignBySigstorePrivateKeyFile string // If non-empty, asks for a signature to be added during the copy, using a sigstore private key file at the provided path.
|
|
SignSigstorePrivateKeyPassphrase []byte // Passphrase to use when signing with `SignBySigstorePrivateKeyFile`.
|
|
SignIdentity reference.Named // Identify to use when signing, defaults to the docker reference of the destination
|
|
|
|
ReportWriter io.Writer
|
|
SourceCtx *types.SystemContext
|
|
DestinationCtx *types.SystemContext
|
|
ProgressInterval time.Duration // time to wait between reports to signal the progress channel
|
|
Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset.
|
|
|
|
// Preserve digests, and fail if we cannot.
|
|
PreserveDigests bool
|
|
// manifest MIME type of image set by user. "" is default and means use the autodetection to the manifest MIME type
|
|
ForceManifestMIMEType string
|
|
ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list
|
|
Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself
|
|
// Give priority to pulling gzip images if multiple images are present when configured to OptionalBoolTrue,
|
|
// prefers the best compression if this is configured as OptionalBoolFalse. Choose automatically (and the choice may change over time)
|
|
// if this is set to OptionalBoolUndefined (which is the default behavior, and recommended for most callers).
|
|
// This only affects CopySystemImage.
|
|
PreferGzipInstances types.OptionalBool
|
|
|
|
// If OciEncryptConfig is non-nil, it indicates that an image should be encrypted.
|
|
// The encryption options is derived from the construction of EncryptConfig object.
|
|
OciEncryptConfig *encconfig.EncryptConfig
|
|
// OciEncryptLayers represents the list of layers to encrypt.
|
|
// If nil, don't encrypt any layers.
|
|
// If non-nil and len==0, denotes encrypt all layers.
|
|
// integers in the slice represent 0-indexed layer indices, with support for negative
|
|
// indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer.
|
|
OciEncryptLayers *[]int
|
|
// OciDecryptConfig contains the config that can be used to decrypt an image if it is
|
|
// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
|
|
OciDecryptConfig *encconfig.DecryptConfig
|
|
|
|
// A weighted semaphore to limit the amount of concurrently copied layers and configs. Applies to all copy operations using the semaphore. If set, MaxParallelDownloads is ignored.
|
|
ConcurrentBlobCopiesSemaphore *semaphore.Weighted
|
|
|
|
// MaxParallelDownloads indicates the maximum layers to pull at the same time. Applies to a single copy operation. A reasonable default is used if this is left as 0. Ignored if ConcurrentBlobCopiesSemaphore is set.
|
|
MaxParallelDownloads uint
|
|
|
|
// When OptimizeDestinationImageAlreadyExists is set, optimize the copy assuming that the destination image already
|
|
// exists (and is equivalent). Making the eventual (no-op) copy more performant for this case. Enabling the option
|
|
// is slightly pessimistic if the destination image doesn't exist, or is not equivalent.
|
|
OptimizeDestinationImageAlreadyExists bool
|
|
|
|
// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
|
|
// to not indicate "nondistributable".
|
|
DownloadForeignLayers bool
|
|
}
|
|
|
|
// copier allows us to keep track of diffID values for blobs, and other
|
|
// data shared across one or more images in a possible manifest list.
|
|
// The owner must call close() when done.
|
|
type copier struct {
|
|
dest private.ImageDestination
|
|
rawSource private.ImageSource
|
|
reportWriter io.Writer
|
|
progressOutput io.Writer
|
|
progressInterval time.Duration
|
|
progress chan types.ProgressProperties
|
|
blobInfoCache internalblobinfocache.BlobInfoCache2
|
|
ociDecryptConfig *encconfig.DecryptConfig
|
|
ociEncryptConfig *encconfig.EncryptConfig
|
|
concurrentBlobCopiesSemaphore *semaphore.Weighted // Limits the amount of concurrently copied blobs
|
|
downloadForeignLayers bool
|
|
signers []*signer.Signer // Signers to use to create new signatures for the image
|
|
signersToClose []*signer.Signer // Signers that should be closed when this copier is destroyed.
|
|
}
|
|
|
|
// Image copies image from srcRef to destRef, using policyContext to validate
|
|
// source image admissibility. It returns the manifest which was written to
|
|
// the new copy of the image.
|
|
func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (copiedManifest []byte, retErr error) {
|
|
// NOTE this function uses an output parameter for the error return value.
|
|
// Setting this and returning is the ideal way to return an error.
|
|
//
|
|
// the defers in this routine will wrap the error return with its own errors
|
|
// which can be valuable context in the middle of a multi-streamed copy.
|
|
if options == nil {
|
|
options = &Options{}
|
|
}
|
|
|
|
if err := validateImageListSelection(options.ImageListSelection); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
reportWriter := io.Discard
|
|
|
|
if options.ReportWriter != nil {
|
|
reportWriter = options.ReportWriter
|
|
}
|
|
|
|
publicDest, err := destRef.NewImageDestination(ctx, options.DestinationCtx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initializing destination %s: %w", transports.ImageName(destRef), err)
|
|
}
|
|
dest := imagedestination.FromPublic(publicDest)
|
|
defer func() {
|
|
if err := dest.Close(); err != nil {
|
|
if retErr != nil {
|
|
retErr = fmt.Errorf(" (dest: %v): %w", err, retErr)
|
|
} else {
|
|
retErr = fmt.Errorf(" (dest: %v)", err)
|
|
}
|
|
}
|
|
}()
|
|
|
|
publicRawSource, err := srcRef.NewImageSource(ctx, options.SourceCtx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initializing source %s: %w", transports.ImageName(srcRef), err)
|
|
}
|
|
rawSource := imagesource.FromPublic(publicRawSource)
|
|
defer func() {
|
|
if err := rawSource.Close(); err != nil {
|
|
if retErr != nil {
|
|
retErr = fmt.Errorf(" (src: %v): %w", err, retErr)
|
|
} else {
|
|
retErr = fmt.Errorf(" (src: %v)", err)
|
|
}
|
|
}
|
|
}()
|
|
|
|
// If reportWriter is not a TTY (e.g., when piping to a file), do not
|
|
// print the progress bars to avoid long and hard to parse output.
|
|
// Instead use printCopyInfo() to print single line "Copying ..." messages.
|
|
progressOutput := reportWriter
|
|
if !isTTY(reportWriter) {
|
|
progressOutput = io.Discard
|
|
}
|
|
|
|
c := &copier{
|
|
dest: dest,
|
|
rawSource: rawSource,
|
|
reportWriter: reportWriter,
|
|
progressOutput: progressOutput,
|
|
progressInterval: options.ProgressInterval,
|
|
progress: options.Progress,
|
|
// FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx.
|
|
// For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually
|
|
// we might want to add a separate CommonCtx — or would that be too confusing?
|
|
blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)),
|
|
ociDecryptConfig: options.OciDecryptConfig,
|
|
ociEncryptConfig: options.OciEncryptConfig,
|
|
downloadForeignLayers: options.DownloadForeignLayers,
|
|
}
|
|
defer c.close()
|
|
|
|
// Set the concurrentBlobCopiesSemaphore if we can copy layers in parallel.
|
|
if dest.HasThreadSafePutBlob() && rawSource.HasThreadSafeGetBlob() {
|
|
c.concurrentBlobCopiesSemaphore = options.ConcurrentBlobCopiesSemaphore
|
|
if c.concurrentBlobCopiesSemaphore == nil {
|
|
max := options.MaxParallelDownloads
|
|
if max == 0 {
|
|
max = maxParallelDownloads
|
|
}
|
|
c.concurrentBlobCopiesSemaphore = semaphore.NewWeighted(int64(max))
|
|
}
|
|
} else {
|
|
c.concurrentBlobCopiesSemaphore = semaphore.NewWeighted(int64(1))
|
|
if options.ConcurrentBlobCopiesSemaphore != nil {
|
|
if err := options.ConcurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil {
|
|
return nil, fmt.Errorf("acquiring semaphore for concurrent blob copies: %w", err)
|
|
}
|
|
defer options.ConcurrentBlobCopiesSemaphore.Release(1)
|
|
}
|
|
}
|
|
|
|
if err := c.setupSigners(options); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
unparsedToplevel := image.UnparsedInstance(rawSource, nil)
|
|
multiImage, err := isMultiImage(ctx, unparsedToplevel)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("determining manifest MIME type for %s: %w", transports.ImageName(srcRef), err)
|
|
}
|
|
|
|
if !multiImage {
|
|
// The simple case: just copy a single image.
|
|
if copiedManifest, _, _, err = c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedToplevel, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
} else if options.ImageListSelection == CopySystemImage {
|
|
// This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that
|
|
// matches the current system to copy, and copy it.
|
|
mfest, manifestType, err := unparsedToplevel.Manifest(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading manifest for %s: %w", transports.ImageName(srcRef), err)
|
|
}
|
|
manifestList, err := internalManifest.ListFromBlob(mfest, manifestType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing primary manifest as list for %s: %w", transports.ImageName(srcRef), err)
|
|
}
|
|
instanceDigest, err := manifestList.ChooseInstanceByCompression(options.SourceCtx, options.PreferGzipInstances) // try to pick one that matches options.SourceCtx
|
|
if err != nil {
|
|
return nil, fmt.Errorf("choosing an image from manifest list %s: %w", transports.ImageName(srcRef), err)
|
|
}
|
|
logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest)
|
|
unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
|
|
|
|
if copiedManifest, _, _, err = c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, nil); err != nil {
|
|
return nil, fmt.Errorf("copying system image from manifest list: %w", err)
|
|
}
|
|
} else { /* options.ImageListSelection == CopyAllImages or options.ImageListSelection == CopySpecificImages, */
|
|
// If we were asked to copy multiple images and can't, that's an error.
|
|
if !supportsMultipleImages(c.dest) {
|
|
return nil, fmt.Errorf("copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name())
|
|
}
|
|
// Copy some or all of the images.
|
|
switch options.ImageListSelection {
|
|
case CopyAllImages:
|
|
logrus.Debugf("Source is a manifest list; copying all instances")
|
|
case CopySpecificImages:
|
|
logrus.Debugf("Source is a manifest list; copying some instances")
|
|
}
|
|
if copiedManifest, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := c.dest.Commit(ctx, unparsedToplevel); err != nil {
|
|
return nil, fmt.Errorf("committing the finished image: %w", err)
|
|
}
|
|
|
|
return copiedManifest, nil
|
|
}
|
|
|
|
// Printf writes a formatted string to c.reportWriter.
|
|
// Note that the method name Printf is not entirely arbitrary: (go tool vet)
|
|
// has a built-in list of functions/methods (whatever object they are for)
|
|
// which have their format strings checked; for other names we would have
|
|
// to pass a parameter to every (go tool vet) invocation.
|
|
func (c *copier) Printf(format string, a ...any) {
|
|
fmt.Fprintf(c.reportWriter, format, a...)
|
|
}
|
|
|
|
// close tears down state owned by copier.
|
|
func (c *copier) close() {
|
|
for i, s := range c.signersToClose {
|
|
if err := s.Close(); err != nil {
|
|
logrus.Warnf("Error closing per-copy signer %d: %v", i+1, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value
|
|
func validateImageListSelection(selection ImageListSelection) error {
|
|
switch selection {
|
|
case CopySystemImage, CopyAllImages, CopySpecificImages:
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("Invalid value for options.ImageListSelection: %d", selection)
|
|
}
|
|
}
|
|
|
|
// Checks if the destination supports accepting multiple images by checking if it can support
|
|
// manifest types that are lists of other manifests.
|
|
func supportsMultipleImages(dest types.ImageDestination) bool {
|
|
mtypes := dest.SupportedManifestMIMETypes()
|
|
if len(mtypes) == 0 {
|
|
// Anything goes!
|
|
return true
|
|
}
|
|
return slices.ContainsFunc(mtypes, manifest.MIMETypeIsMultiImage)
|
|
}
|
|
|
|
// isTTY returns true if the io.Writer is a file and a tty.
|
|
func isTTY(w io.Writer) bool {
|
|
if f, ok := w.(*os.File); ok {
|
|
return term.IsTerminal(int(f.Fd()))
|
|
}
|
|
return false
|
|
}
|