From 47ac96155f1573e1bbf7804461f6850ff67e8747 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 28 Jun 2017 17:07:58 -0400 Subject: [PATCH] Use configured registries to resolve image names When locating an image for pulling, inspection, or pushing, if we're given an image name that doesn't include a domain/registry, try building a set of candidate names using the configured registries as domains, and then pull/inspect/push using the first of those names that works. If a name that we're given corresponds to a prefix of the ID of a local image, skip completion and use the ID directly instead. Signed-off-by: Nalin Dahyabhai Closes: #360 Approved by: rhatdan --- buildah.go | 4 + cmd/buildah/common.go | 13 +- cmd/buildah/from.go | 1 - cmd/buildah/images.go | 4 +- cmd/buildah/inspect.go | 9 +- cmd/buildah/main.go | 8 + cmd/buildah/rmi.go | 4 +- commit.go | 14 +- common.go | 5 +- docs/buildah.md | 18 +- import.go | 36 ++-- new.go | 188 +++++++++++------- pull.go | 41 ++-- tests/byid.bats | 101 ++++++++++ tests/helpers.bash | 2 +- tests/registries.bats | 57 ++++++ tests/registries.conf | 25 +++ util/util.go | 114 ++++++++++- .../pkg/sysregistries/system_registries.go | 86 ++++++++ 19 files changed, 594 insertions(+), 136 deletions(-) create mode 100644 tests/byid.bats create mode 100644 tests/registries.bats create mode 100644 tests/registries.conf create mode 100644 vendor/github.com/containers/image/pkg/sysregistries/system_registries.go diff --git a/buildah.go b/buildah.go index 064a3a06a..8d83e3b1e 100644 --- a/buildah.go +++ b/buildah.go @@ -159,6 +159,10 @@ type ImportFromImageOptions struct { // specified, indicating that the shared, system-wide default policy // should be used. SignaturePolicyPath string + // github.com/containers/image/types SystemContext to hold information + // about which registries we should check for completing image names + // that don't include a domain portion. + SystemContext *types.SystemContext } // NewBuilder creates a new build container. diff --git a/cmd/buildah/common.go b/cmd/buildah/common.go index 2fddeef2b..d816ef8c9 100644 --- a/cmd/buildah/common.go +++ b/cmd/buildah/common.go @@ -65,9 +65,10 @@ func openBuilders(store storage.Store) (builders []*buildah.Builder, err error) return buildah.OpenAllBuilders(store) } -func openImage(store storage.Store, name string) (builder *buildah.Builder, err error) { +func openImage(sc *types.SystemContext, store storage.Store, name string) (builder *buildah.Builder, err error) { options := buildah.ImportFromImageOptions{ - Image: name, + Image: name, + SystemContext: sc, } builder, err = buildah.ImportBuilderFromImage(store, options) if err != nil { @@ -82,7 +83,7 @@ func openImage(store storage.Store, name string) (builder *buildah.Builder, err func getDateAndDigestAndSize(image storage.Image, store storage.Store) (time.Time, string, int64, error) { created := time.Time{} is.Transport.SetStore(store) - storeRef, err := is.Transport.ParseStoreReference(store, "@"+image.ID) + storeRef, err := is.Transport.ParseStoreReference(store, image.ID) if err != nil { return created, "", -1, err } @@ -136,6 +137,12 @@ func systemContextFromOptions(c *cli.Context) (*types.SystemContext, error) { if c.IsSet("authfile") { ctx.AuthFilePath = c.String("authfile") } + if c.GlobalIsSet("registries-conf") { + ctx.SystemRegistriesConfPath = c.GlobalString("registries-conf") + } + if c.GlobalIsSet("registries-conf-dir") { + ctx.RegistriesDirPath = c.GlobalString("registries-conf-dir") + } return ctx, nil } diff --git a/cmd/buildah/from.go b/cmd/buildah/from.go index 0d274c6aa..8a421c9e8 100644 --- a/cmd/buildah/from.go +++ b/cmd/buildah/from.go @@ -63,7 +63,6 @@ var ( ) func fromCmd(c *cli.Context) error { - args := c.Args() if len(args) == 0 { return errors.Errorf("an image name (or \"scratch\") must be specified") diff --git a/cmd/buildah/images.go b/cmd/buildah/images.go index 275acd105..c7610a5a1 100644 --- a/cmd/buildah/images.go +++ b/cmd/buildah/images.go @@ -178,7 +178,7 @@ func setFilterDate(store storage.Store, images []storage.Image, imgName string) for _, name := range image.Names { if matchesReference(name, imgName) { // Set the date to this image - ref, err := is.Transport.ParseStoreReference(store, "@"+image.ID) + ref, err := is.Transport.ParseStoreReference(store, image.ID) if err != nil { return time.Time{}, fmt.Errorf("error parsing reference to image %q: %v", image.ID, err) } @@ -290,7 +290,7 @@ func matchesDangling(name string, dangling string) bool { } func matchesLabel(image storage.Image, store storage.Store, label string) bool { - storeRef, err := is.Transport.ParseStoreReference(store, "@"+image.ID) + storeRef, err := is.Transport.ParseStoreReference(store, image.ID) if err != nil { return false } diff --git a/cmd/buildah/inspect.go b/cmd/buildah/inspect.go index cadcfc42e..bd2843f4f 100644 --- a/cmd/buildah/inspect.go +++ b/cmd/buildah/inspect.go @@ -56,6 +56,11 @@ func inspectCmd(c *cli.Context) error { return err } + systemContext, err := systemContextFromOptions(c) + if err != nil { + return errors.Wrapf(err, "error building system context") + } + format := defaultFormat if c.String("format") != "" { format = c.String("format") @@ -76,13 +81,13 @@ func inspectCmd(c *cli.Context) error { if c.IsSet("type") { return errors.Wrapf(err, "error reading build container %q", name) } - builder, err = openImage(store, name) + builder, err = openImage(systemContext, store, name) if err != nil { return errors.Wrapf(err, "error reading build object %q", name) } } case inspectTypeImage: - builder, err = openImage(store, name) + builder, err = openImage(systemContext, store, name) if err != nil { return errors.Wrapf(err, "error reading image %q", name) } diff --git a/cmd/buildah/main.go b/cmd/buildah/main.go index f9537a191..5305fad02 100644 --- a/cmd/buildah/main.go +++ b/cmd/buildah/main.go @@ -33,6 +33,14 @@ func main() { Name: "debug", Usage: "print debugging information", }, + cli.StringFlag{ + Name: "registries-conf", + Usage: "path to registries.conf file (not usually used)", + }, + cli.StringFlag{ + Name: "registries-conf-dir", + Usage: "path to registries.conf.d directory (not usually used)", + }, cli.StringFlag{ Name: "root", Usage: "storage root dir", diff --git a/cmd/buildah/rmi.go b/cmd/buildah/rmi.go index cc9f3a53b..01102cf0a 100644 --- a/cmd/buildah/rmi.go +++ b/cmd/buildah/rmi.go @@ -210,12 +210,12 @@ func storageImageID(store storage.Store, id string) (types.ImageReference, error if img, err := store.Image(id); err == nil && img != nil { imageID = img.ID } - if ref, err := is.Transport.ParseStoreReference(store, "@"+imageID); err == nil { + if ref, err := is.Transport.ParseStoreReference(store, imageID); err == nil { if img, err2 := ref.NewImage(nil); err2 == nil { img.Close() return ref, nil } return nil, errors.Wrapf(err, "error confirming presence of storage image reference %q", transports.ImageName(ref)) } - return nil, errors.Wrapf(err, "error parsing %q as a storage image reference: %v", "@"+id) + return nil, errors.Wrapf(err, "error parsing %q as a storage image reference: %v", id) } diff --git a/commit.go b/commit.go index b103d9d23..b1d8d958f 100644 --- a/commit.go +++ b/commit.go @@ -180,7 +180,7 @@ func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReferenc // configuration, to a new image in the specified location, and if we know how, // add any additional tags that were specified. func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error { - policy, err := signature.DefaultPolicy(getSystemContext(options.SignaturePolicyPath)) + policy, err := signature.DefaultPolicy(getSystemContext(options.SystemContext, options.SignaturePolicyPath)) if err != nil { return errors.Wrapf(err, "error obtaining default signature policy") } @@ -190,7 +190,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error } defer func() { if err2 := policyContext.Destroy(); err2 != nil { - logrus.Debugf("error destroying signature polcy context: %v", err2) + logrus.Debugf("error destroying signature policy context: %v", err2) } }() // Check if we're keeping everything in local storage. If so, we can take certain shortcuts. @@ -208,7 +208,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error } } else { // Copy only the most recent layer, the configuration, and the manifest. - err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath), options.Compression) + err = b.shallowCopy(dest, src, getSystemContext(options.SystemContext, options.SignaturePolicyPath), options.Compression) if err != nil { return errors.Wrapf(err, "error copying layer and metadata") } @@ -234,7 +234,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error // Push copies the contents of the image to a new location. func Push(image string, dest types.ImageReference, options PushOptions) error { - systemContext := getSystemContext(options.SignaturePolicyPath) + systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath) policy, err := signature.DefaultPolicy(systemContext) if err != nil { return errors.Wrapf(err, "error obtaining default signature policy") @@ -246,11 +246,7 @@ func Push(image string, dest types.ImageReference, options PushOptions) error { // Look up the image. src, err := is.Transport.ParseStoreReference(options.Store, image) if err != nil { - src2, err2 := is.Transport.ParseStoreReference(options.Store, "@"+image) - if err2 != nil { - return errors.Wrapf(err, "error parsing reference to image %q", image) - } - src = src2 + return errors.Wrapf(err, "error parsing reference to image %q", image) } // Copy everything. err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter, nil, options.SystemContext, options.ManifestType)) diff --git a/common.go b/common.go index acde45f7b..18c960003 100644 --- a/common.go +++ b/common.go @@ -16,8 +16,11 @@ func getCopyOptions(reportWriter io.Writer, sourceSystemContext *types.SystemCon } } -func getSystemContext(signaturePolicyPath string) *types.SystemContext { +func getSystemContext(defaults *types.SystemContext, signaturePolicyPath string) *types.SystemContext { sc := &types.SystemContext{} + if defaults != nil { + *sc = *defaults + } if signaturePolicyPath != "" { sc.SignaturePolicyPath = signaturePolicyPath } diff --git a/docs/buildah.md b/docs/buildah.md index 39960f91b..6f48bd10b 100644 --- a/docs/buildah.md +++ b/docs/buildah.md @@ -24,12 +24,28 @@ Print debugging information **--default-mounts-file** -path to default mounts file (default path: "/usr/share/containers/mounts.conf") +Path to default mounts file (default path: "/usr/share/containers/mounts.conf") **--help, -h** Show help +**--registries-conf** *path* + +Pathname of the configuration file which specifies which registries should be +consulted when completing image names which do not include a registry or domain +portion. It is not recommended that this option be used, as the default +behavior of using the system-wide configuration +(*/etc/containers/registries.conf*) is most often preferred. + +**--registries-conf-dir** *path* + +Pathname of the directory which contains configuration snippets which specify +registries which should be consulted when completing image names which do not +include a registry or domain portion. It is not recommended that this option +be used, as the default behavior of using the system-wide configuration +(*/etc/containers/registries.d*) is most often preferred. + **--root** **value** Storage root dir (default: "/var/lib/containers/storage") diff --git a/import.go b/import.go index 51ef13d17..73a06b9d8 100644 --- a/import.go +++ b/import.go @@ -16,9 +16,9 @@ func importBuilderDataFromImage(store storage.Store, systemContext *types.System imageName := "" if imageID != "" { - ref, err := is.Transport.ParseStoreReference(store, "@"+imageID) + ref, err := is.Transport.ParseStoreReference(store, imageID) if err != nil { - return nil, errors.Wrapf(err, "no such image %q", "@"+imageID) + return nil, errors.Wrapf(err, "no such image %q", imageID) } src, err2 := ref.NewImage(systemContext) if err2 != nil { @@ -68,7 +68,7 @@ func importBuilder(store storage.Store, options ImportOptions) (*Builder, error) return nil, err } - systemContext := getSystemContext(options.SignaturePolicyPath) + systemContext := getSystemContext(&types.SystemContext{}, options.SignaturePolicyPath) builder, err := importBuilderDataFromImage(store, systemContext, c.ImageID, options.Container, c.ID) if err != nil { @@ -95,21 +95,27 @@ func importBuilder(store storage.Store, options ImportOptions) (*Builder, error) } func importBuilderFromImage(store storage.Store, options ImportFromImageOptions) (*Builder, error) { + var img *storage.Image + var err error + if options.Image == "" { return nil, errors.Errorf("image name must be specified") } - img, err := util.FindImage(store, options.Image) - if err != nil { - return nil, errors.Wrapf(err, "error locating image %q for importing settings", options.Image) + systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath) + + for _, image := range util.ResolveName(options.Image, "", systemContext, store) { + img, err = util.FindImage(store, image) + if err != nil { + continue + } + + builder, err2 := importBuilderDataFromImage(store, systemContext, img.ID, "", "") + if err2 != nil { + return nil, errors.Wrapf(err2, "error importing build settings from image %q", options.Image) + } + + return builder, nil } - - systemContext := getSystemContext(options.SignaturePolicyPath) - - builder, err := importBuilderDataFromImage(store, systemContext, img.ID, "", "") - if err != nil { - return nil, errors.Wrapf(err, "error importing build settings from image %q", options.Image) - } - - return builder, nil + return nil, errors.Wrapf(err, "error locating image %q for importing settings", options.Image) } diff --git a/new.go b/new.go index 0fc81844a..21bc51480 100644 --- a/new.go +++ b/new.go @@ -14,6 +14,7 @@ import ( "github.com/opencontainers/selinux/go-selinux/label" "github.com/openshift/imagebuilder" "github.com/pkg/errors" + "github.com/projectatomic/buildah/util" "github.com/sirupsen/logrus" ) @@ -26,6 +27,10 @@ const ( // can't find one in the local Store, in order to generate a source // reference for the image that we can then copy to the local Store. DefaultTransport = "docker://" + + // minimumTruncatedIDLength is the minimum length of an identifier that + // we'll accept as possibly being a truncated image ID. + minimumTruncatedIDLength = 3 ) func reserveSELinuxLabels(store storage.Store, id string) error { @@ -58,91 +63,150 @@ func reserveSELinuxLabels(store storage.Store, id string) error { return nil } +func pullAndFindImage(store storage.Store, imageName string, options BuilderOptions, sc *types.SystemContext) (*storage.Image, types.ImageReference, error) { + ref, err := pullImage(store, imageName, options, sc) + if err != nil { + logrus.Debugf("error pulling image %q: %v", imageName, err) + return nil, nil, err + } + img, err := is.Transport.GetStoreImage(store, ref) + if err != nil { + logrus.Debugf("error reading pulled image %q: %v", imageName, err) + return nil, nil, err + } + return img, ref, nil +} + +func imageNamePrefix(imageName string) string { + prefix := imageName + s := strings.Split(imageName, "/") + if len(s) > 0 { + prefix = s[len(s)-1] + } + s = strings.Split(prefix, ":") + if len(s) > 0 { + prefix = s[0] + } + s = strings.Split(prefix, "@") + if len(s) > 0 { + prefix = s[0] + } + return prefix +} + +func imageManifestAndConfig(ref types.ImageReference, systemContext *types.SystemContext) (manifest, config []byte, err error) { + if ref != nil { + src, err := ref.NewImage(systemContext) + if err != nil { + return nil, nil, errors.Wrapf(err, "error instantiating image for %q", transports.ImageName(ref)) + } + defer src.Close() + config, err := src.ConfigBlob() + if err != nil { + return nil, nil, errors.Wrapf(err, "error reading image configuration for %q", transports.ImageName(ref)) + } + manifest, _, err := src.Manifest() + if err != nil { + return nil, nil, errors.Wrapf(err, "error reading image manifest for %q", transports.ImageName(ref)) + } + return manifest, config, nil + } + return nil, nil, nil +} + func newBuilder(store storage.Store, options BuilderOptions) (*Builder, error) { var ref types.ImageReference var img *storage.Image - manifest := []byte{} - config := []byte{} + var err error + var manifest []byte + var config []byte if options.FromImage == BaseImageFakeName { options.FromImage = "" } - image := options.FromImage - if options.Transport == "" { options.Transport = DefaultTransport } - systemContext := getSystemContext(options.SignaturePolicyPath) + systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath) + + for _, image := range util.ResolveName(options.FromImage, options.Registry, systemContext, store) { + if len(image) >= minimumTruncatedIDLength { + if img, err = store.Image(image); err == nil && img != nil && strings.HasPrefix(img.ID, image) { + if ref, err = is.Transport.ParseStoreReference(store, img.ID); err != nil { + return nil, errors.Wrapf(err, "error parsing reference to image %q", img.ID) + } + break + } + } - imageID := "" - if image != "" { - var err error if options.PullPolicy == PullAlways { - pulledReference, err2 := pullImage(store, options, systemContext) + pulledImg, pulledReference, err2 := pullAndFindImage(store, image, options, systemContext) if err2 != nil { - return nil, errors.Wrapf(err2, "error pulling image %q", image) + logrus.Debugf("error pulling and reading image %q: %v", image, err2) + continue } ref = pulledReference + img = pulledImg + break } - if ref == nil { - srcRef, err2 := alltransports.ParseImageName(image) - if err2 != nil { - srcRef2, err3 := alltransports.ParseImageName(options.Registry + image) - if err3 != nil { - srcRef3, err4 := alltransports.ParseImageName(options.Transport + options.Registry + image) - if err4 != nil { - return nil, errors.Wrapf(err4, "error parsing image name %q", options.Transport+options.Registry+image) - } - srcRef2 = srcRef3 - } - srcRef = srcRef2 - } - destImage, err2 := localImageNameForReference(store, srcRef) - if err2 != nil { - return nil, errors.Wrapf(err2, "error computing local image name for %q", transports.ImageName(srcRef)) + srcRef, err2 := alltransports.ParseImageName(image) + if err2 != nil { + if options.Transport == "" { + logrus.Debugf("error parsing image name %q: %v", image, err2) + continue } - if destImage == "" { - return nil, errors.Errorf("error computing local image name for %q", transports.ImageName(srcRef)) + srcRef2, err3 := alltransports.ParseImageName(options.Transport + image) + if err3 != nil { + logrus.Debugf("error parsing image name %q: %v", image, err2) + continue } + srcRef = srcRef2 + } - ref, err = is.Transport.ParseStoreReference(store, destImage) - if err != nil { - return nil, errors.Wrapf(err, "error parsing reference to image %q", destImage) - } + destImage, err2 := localImageNameForReference(store, srcRef) + if err2 != nil { + return nil, errors.Wrapf(err2, "error computing local image name for %q", transports.ImageName(srcRef)) + } + if destImage == "" { + return nil, errors.Errorf("error computing local image name for %q", transports.ImageName(srcRef)) + } - image = destImage + ref, err = is.Transport.ParseStoreReference(store, destImage) + if err != nil { + return nil, errors.Wrapf(err, "error parsing reference to image %q", destImage) } img, err = is.Transport.GetStoreImage(store, ref) if err != nil { if errors.Cause(err) == storage.ErrImageUnknown && options.PullPolicy != PullIfMissing { - return nil, errors.Wrapf(err, "no such image %q", transports.ImageName(ref)) + logrus.Debugf("no such image %q: %v", transports.ImageName(ref), err) + continue } - ref2, err2 := pullImage(store, options, systemContext) + pulledImg, pulledReference, err2 := pullAndFindImage(store, image, options, systemContext) if err2 != nil { - return nil, errors.Wrapf(err2, "error pulling image %q", image) + logrus.Debugf("error pulling and reading image %q: %v", image, err2) + continue } - ref = ref2 - img, err = is.Transport.GetStoreImage(store, ref) + ref = pulledReference + img = pulledImg } - if err != nil { - return nil, errors.Wrapf(err, "no such image %q", transports.ImageName(ref)) + break + } + + if options.FromImage != "" && (ref == nil || img == nil) { + return nil, errors.Wrapf(storage.ErrImageUnknown, "no such image %q", options.FromImage) + } + image := options.FromImage + imageID := "" + if img != nil { + if len(img.Names) > 0 { + image = img.Names[0] } imageID = img.ID - src, err := ref.NewImage(systemContext) - if err != nil { - return nil, errors.Wrapf(err, "error instantiating image for %q", transports.ImageName(ref)) - } - defer src.Close() - config, err = src.ConfigBlob() - if err != nil { - return nil, errors.Wrapf(err, "error reading image configuration for %q", transports.ImageName(ref)) - } - manifest, _, err = src.Manifest() - if err != nil { - return nil, errors.Wrapf(err, "error reading image manifest for %q", transports.ImageName(ref)) - } + } + if manifest, config, err = imageManifestAndConfig(ref, systemContext); err != nil { + return nil, errors.Wrapf(err, "error reading data from image %q", transports.ImageName(ref)) } name := "working-container" @@ -151,20 +215,7 @@ func newBuilder(store storage.Store, options BuilderOptions) (*Builder, error) { } else { var err2 error if image != "" { - prefix := image - s := strings.Split(prefix, "/") - if len(s) > 0 { - prefix = s[len(s)-1] - } - s = strings.Split(prefix, ":") - if len(s) > 0 { - prefix = s[0] - } - s = strings.Split(prefix, "@") - if len(s) > 0 { - prefix = s[0] - } - name = prefix + "-" + name + name = imageNamePrefix(image) + "-" + name } suffix := 1 tmpName := name @@ -177,6 +228,7 @@ func newBuilder(store storage.Store, options BuilderOptions) (*Builder, error) { } name = tmpName } + coptions := storage.ContainerOptions{} container, err := store.CreateContainer("", []string{name}, imageID, "", "", &coptions) if err != nil { @@ -191,7 +243,7 @@ func newBuilder(store storage.Store, options BuilderOptions) (*Builder, error) { } }() - if err := reserveSELinuxLabels(store, container.ID); err != nil { + if err = reserveSELinuxLabels(store, container.ID); err != nil { return nil, err } processLabel, mountLabel, err := label.InitLabels(nil) diff --git a/pull.go b/pull.go index 993285a69..c8113aa58 100644 --- a/pull.go +++ b/pull.go @@ -53,27 +53,17 @@ func localImageNameForReference(store storage.Store, srcRef types.ImageReference return name, nil } -func pullImage(store storage.Store, options BuilderOptions, sc *types.SystemContext) (types.ImageReference, error) { - name := options.FromImage - - spec := name - if options.Registry != "" { - spec = options.Registry + spec - } - spec2 := spec - if options.Transport != "" { - spec2 = options.Transport + spec - } - - srcRef, err := alltransports.ParseImageName(name) +func pullImage(store storage.Store, imageName string, options BuilderOptions, sc *types.SystemContext) (types.ImageReference, error) { + spec := imageName + srcRef, err := alltransports.ParseImageName(spec) if err != nil { + if options.Transport == "" { + return nil, errors.Wrapf(err, "error parsing image name %q", spec) + } + spec = options.Transport + spec srcRef2, err2 := alltransports.ParseImageName(spec) if err2 != nil { - srcRef3, err3 := alltransports.ParseImageName(spec2) - if err3 != nil { - return nil, errors.Wrapf(err3, "error parsing image name %q", spec2) - } - srcRef2 = srcRef3 + return nil, errors.Wrapf(err2, "error parsing image name %q", spec) } srcRef = srcRef2 } @@ -91,6 +81,12 @@ func pullImage(store storage.Store, options BuilderOptions, sc *types.SystemCont return nil, errors.Wrapf(err, "error parsing image name %q", destName) } + img, err := srcRef.NewImageSource(sc) + if err != nil { + return nil, errors.Wrapf(err, "error initializing %q as an image source", spec) + } + img.Close() + policy, err := signature.DefaultPolicy(sc) if err != nil { return nil, errors.Wrapf(err, "error obtaining default signature policy") @@ -103,12 +99,15 @@ func pullImage(store storage.Store, options BuilderOptions, sc *types.SystemCont defer func() { if err2 := policyContext.Destroy(); err2 != nil { - logrus.Debugf("error destroying signature polcy context: %v", err2) + logrus.Debugf("error destroying signature policy context: %v", err2) } }() - logrus.Debugf("copying %q to %q", spec, name) + logrus.Debugf("copying %q to %q", spec, destName) err = cp.Image(policyContext, destRef, srcRef, getCopyOptions(options.ReportWriter, options.SystemContext, nil, "")) - return destRef, err + if err == nil { + return destRef, nil + } + return nil, err } diff --git a/tests/byid.bats b/tests/byid.bats new file mode 100644 index 000000000..1bc91d1cd --- /dev/null +++ b/tests/byid.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats + +load helpers + +@test "from-by-id" { + image=busybox + + # Pull down the image, if we have to. + cid=$(buildah --debug=false from --pull --signature-policy ${TESTSDIR}/policy.json $image) + [ $? -eq 0 ] + [ $(wc -l <<< "$cid") -eq 1 ] + buildah rm $cid + + # Get the image's ID. + run buildah --debug=false images -q $image + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + iid="$output" + + # Use the image's ID to create a container. + run buildah --debug=false from --pull --signature-policy ${TESTSDIR}/policy.json ${iid} + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + cid="$output" + buildah rm $cid + + # Use a truncated form of the image's ID to create a container. + run buildah --debug=false from --pull --signature-policy ${TESTSDIR}/policy.json ${iid:0:6} + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + cid="$output" + buildah rm $cid + + buildah rmi $iid +} + +@test "inspect-by-id" { + image=busybox + + # Pull down the image, if we have to. + cid=$(buildah --debug=false from --pull --signature-policy ${TESTSDIR}/policy.json $image) + [ $? -eq 0 ] + [ $(wc -l <<< "$cid") -eq 1 ] + buildah rm $cid + + # Get the image's ID. + run buildah --debug=false images -q $image + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + iid="$output" + + # Use the image's ID to inspect it. + run buildah --debug=false inspect --type=image ${iid} + echo "$output" + [ $status -eq 0 ] + + # Use a truncated copy of the image's ID to inspect it. + run buildah --debug=false inspect --type=image ${iid:0:6} + echo "$output" + [ $status -eq 0 ] + + buildah rmi $iid +} + +@test "push-by-id" { + for image in busybox kubernetes/pause ; do + echo pulling/pushing image $image + + TARGET=${TESTDIR}/subdir-$(basename $image) + mkdir -p $TARGET $TARGET-truncated + + # Pull down the image, if we have to. + cid=$(buildah --debug=false from --pull --signature-policy ${TESTSDIR}/policy.json $image) + [ $? -eq 0 ] + [ $(wc -l <<< "$cid") -eq 1 ] + buildah rm $cid + + # Get the image's ID. + run buildah --debug=false images -q $IMAGE + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + iid="$output" + + # Use the image's ID to push it. + run buildah push --signature-policy ${TESTSDIR}/policy.json $iid dir:$TARGET + echo "$output" + [ $status -eq 0 ] + + # Use a truncated form of the image's ID to push it. + run buildah push --signature-policy ${TESTSDIR}/policy.json ${iid:0:6} dir:$TARGET-truncated + echo "$output" + [ $status -eq 0 ] + + buildah rmi $iid + done +} diff --git a/tests/helpers.bash b/tests/helpers.bash index 35e9448b0..5971be2ed 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -42,7 +42,7 @@ function createrandom() { } function buildah() { - ${BUILDAH_BINARY} --debug --root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER} "$@" + ${BUILDAH_BINARY} --debug --registries-conf ${TESTSDIR}/registries.conf --root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER} "$@" } function imgtype() { diff --git a/tests/registries.bats b/tests/registries.bats new file mode 100644 index 000000000..f44a34c66 --- /dev/null +++ b/tests/registries.bats @@ -0,0 +1,57 @@ +#!/usr/bin/env bats + +load helpers + +@test "registries" { + registrypair() { + image=$1 + imagename=$2 + + # Clean up. + for id in $(buildah --debug=false containers -q) ; do + buildah rm ${id} + done + for id in $(buildah --debug=false images -q) ; do + buildah rmi ${id} + done + + # Create a container by specifying the image with one name. + buildah from --pull --signature-policy ${TESTSDIR}/policy.json $image + + # Create a container by specifying the image with another name. + buildah from --pull --signature-policy ${TESTSDIR}/policy.json $imagename + + # Get their image IDs. They should be the same one. + lastid= + for cid in $(buildah --debug=false containers -q) ; do + run buildah --debug=false inspect -f "{{.FromImageID}}" $cid + echo "$output" + [ $status -eq 0 ] + [ $(wc -l <<< "$output") -eq 1 ] + if [ "$lastid" != "" ] ; then + [ "$output" = "$lastid" ] + fi + lastid="$output" + done + + # A quick bit of troubleshooting help. + run buildah images + echo "$output" + [ "$iid" = "$nameiid" ] + + # Clean up. + for id in $(buildah --debug=false containers -q) ; do + buildah rm ${id} + done + for id in $(buildah --debug=false images -q) ; do + buildah rmi ${id} + done + } + # Test with pairs of short and fully-qualified names that should be the same image. + registrypair busybox docker.io/busybox + registrypair docker.io/busybox busybox + registrypair busybox docker.io/library/busybox + registrypair docker.io/library/busybox busybox + registrypair fedora-minimal registry.fedoraproject.org/fedora-minimal + registrypair registry.fedoraproject.org/fedora-minimal fedora-minimal +} diff --git a/tests/registries.conf b/tests/registries.conf new file mode 100644 index 000000000..b09ed643f --- /dev/null +++ b/tests/registries.conf @@ -0,0 +1,25 @@ +# This is a system-wide configuration file used to +# keep track of registries for various container backends. +# It adheres to TOML format and does not support recursive +# lists of registries. + +# The default location for this configuration file is /etc/containers/registries.conf. + +# The only valid categories are: 'registries.search', 'registries.insecure', +# and 'registries.block'. + +[registries.search] +registries = ['docker.io', 'registry.fedoraproject.org', 'registry.access.redhat.com'] + +# If you need to access insecure registries, add the registry's fully-qualified name. +# An insecure registry is one that does not have a valid SSL certificate or only does HTTP. +[registries.insecure] +registries = [] + + +# If you need to block pull access from a registry, uncomment the section below +# and add the registries fully-qualified name. +# +# Docker only +[registries.block] +registries = [] diff --git a/util/util.go b/util/util.go index 807994779..7e92e689d 100644 --- a/util/util.go +++ b/util/util.go @@ -1,27 +1,121 @@ package util import ( + "path" + "strings" + "github.com/containers/image/docker/reference" + "github.com/containers/image/pkg/sysregistries" is "github.com/containers/image/storage" + "github.com/containers/image/types" "github.com/containers/storage" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) -// ExpandTags takes unqualified names, parses them as image names, and returns -// the fully expanded result, including a tag. -func ExpandTags(tags []string) ([]string, error) { - expanded := []string{} - for _, tag := range tags { - name, err := reference.ParseNormalizedNamed(tag) +const ( + minimumTruncatedIDLength = 3 +) + +var ( + // RegistryDefaultPathPrefix contains a per-registry listing of default prefixes + // to prepend to image names that only contain a single path component. + RegistryDefaultPathPrefix = map[string]string{ + "index.docker.io": "library", + "docker.io": "library", + } +) + +// ResolveName checks if name is a valid image name, and if that name doesn't include a domain +// portion, returns a list of the names which it might correspond to in the registries. +func ResolveName(name string, firstRegistry string, sc *types.SystemContext, store storage.Store) []string { + if name == "" { + return nil + } + + // Maybe it's a truncated image ID. Don't prepend a registry name, then. + if len(name) >= minimumTruncatedIDLength { + if img, err := store.Image(name); err == nil && img != nil && strings.HasPrefix(img.ID, name) { + // It's a truncated version of the ID of an image that's present in local storage; + // we need to expand the ID. + return []string{img.ID} + } + } + + // If the image name already included a domain component, we're done. + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return []string{name} + } + if named.String() == name { + // Parsing produced the same result, so there was a domain name in there to begin with. + return []string{name} + } + if reference.Domain(named) != "" && RegistryDefaultPathPrefix[reference.Domain(named)] != "" { + // If this domain can cause us to insert something in the middle, check if that happened. + repoPath := reference.Path(named) + domain := reference.Domain(named) + defaultPrefix := RegistryDefaultPathPrefix[reference.Domain(named)] + "/" + if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):]) == name { + // Yup, parsing just inserted a bit in the middle, so there was a domain name there to begin with. + return []string{name} + } + } + + // Figure out the list of registries. + registries, err := sysregistries.GetRegistries(sc) + if err != nil { + logrus.Debugf("unable to complete image name %q: %v", name, err) + return []string{name} + } + if sc.DockerInsecureSkipTLSVerify { + if unverifiedRegistries, err := sysregistries.GetInsecureRegistries(sc); err == nil { + registries = append(registries, unverifiedRegistries...) + } + } + + // Create all of the combinations. Some registries need an additional component added, so + // use our lookaside map to keep track of them. If there are no configured registries, at + // least return the name as it was passed to us. + candidates := []string{} + for _, registry := range append([]string{firstRegistry}, registries...) { + if registry == "" { + continue + } + middle := "" + if prefix, ok := RegistryDefaultPathPrefix[registry]; ok && strings.IndexRune(name, '/') == -1 { + middle = prefix + } + candidate := path.Join(registry, middle, name) + candidates = append(candidates, candidate) + } + if len(candidates) == 0 { + candidates = append(candidates, name) + } + return candidates +} + +// ExpandNames takes unqualified names, parses them as image names, and returns +// the fully expanded result, including a tag. Names which don't include a registry +// name will be marked for the most-preferred registry (i.e., the first one in our +// configuration). +func ExpandNames(names []string) ([]string, error) { + expanded := make([]string, 0, len(names)) + for _, n := range names { + name, err := reference.ParseNormalizedNamed(n) if err != nil { - return nil, errors.Wrapf(err, "error parsing tag %q", tag) + return nil, errors.Wrapf(err, "error parsing name %q", n) } name = reference.TagNameOnly(name) - tag = "" + tag := "" + digest := "" if tagged, ok := name.(reference.NamedTagged); ok { tag = ":" + tagged.Tag() } - expanded = append(expanded, name.Name()+tag) + if digested, ok := name.(reference.Digested); ok { + digest = "@" + digested.Digest().String() + } + expanded = append(expanded, name.Name()+tag+digest) } return expanded, nil } @@ -48,7 +142,7 @@ func FindImage(store storage.Store, image string) (*storage.Image, error) { // AddImageNames adds the specified names to the specified image. func AddImageNames(store storage.Store, image *storage.Image, addNames []string) error { - names, err := ExpandTags(addNames) + names, err := ExpandNames(addNames) if err != nil { return err } diff --git a/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go b/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go new file mode 100644 index 000000000..e5564a2a2 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go @@ -0,0 +1,86 @@ +package sysregistries + +import ( + "github.com/BurntSushi/toml" + "github.com/containers/image/types" + "io/ioutil" + "path/filepath" +) + +// systemRegistriesConfPath is the path to the system-wide registry configuration file +// and is used to add/subtract potential registries for obtaining images. +// You can override this at build time with +// -ldflags '-X github.com/containers/image/sysregistries.systemRegistriesConfPath=$your_path' +var systemRegistriesConfPath = builtinRegistriesConfPath + +// builtinRegistriesConfPath is the path to registry configuration file +// DO NOT change this, instead see systemRegistriesConfPath above. +const builtinRegistriesConfPath = "/etc/containers/registries.conf" + +type registries struct { + Registries []string `toml:"registries"` +} + +type tomlConfig struct { + Registries struct { + Search registries `toml:"search"` + Insecure registries `toml:"insecure"` + Block registries `toml:"block"` + } `toml:"registries"` +} + +// Reads the global registry file from the filesystem. Returns +// a byte array +func readRegistryConf(ctx *types.SystemContext) ([]byte, error) { + dirPath := systemRegistriesConfPath + if ctx != nil { + if ctx.SystemRegistriesConfPath != "" { + dirPath = ctx.SystemRegistriesConfPath + } else if ctx.RootForImplicitAbsolutePaths != "" { + dirPath = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath) + } + } + configBytes, err := ioutil.ReadFile(dirPath) + return configBytes, err +} + +// For mocking in unittests +var readConf = readRegistryConf + +// Loads the registry configuration file from the filesystem and +// then unmarshals it. Returns the unmarshalled object. +func loadRegistryConf(ctx *types.SystemContext) (*tomlConfig, error) { + config := &tomlConfig{} + + configBytes, err := readConf(ctx) + if err != nil { + return nil, err + } + + err = toml.Unmarshal(configBytes, &config) + return config, err +} + +// GetRegistries returns an array of strings that contain the names +// of the registries as defined in the system-wide +// registries file. it returns an empty array if none are +// defined +func GetRegistries(ctx *types.SystemContext) ([]string, error) { + config, err := loadRegistryConf(ctx) + if err != nil { + return nil, err + } + return config.Registries.Search.Registries, nil +} + +// GetInsecureRegistries returns an array of strings that contain the names +// of the insecure registries as defined in the system-wide +// registries file. it returns an empty array if none are +// defined +func GetInsecureRegistries(ctx *types.SystemContext) ([]string, error) { + config, err := loadRegistryConf(ctx) + if err != nil { + return nil, err + } + return config.Registries.Insecure.Registries, nil +}