bump(github.com/containers/imge)

Bump github.com/containers/image to version
5e5b67d6b1cf43cc349128ec3ed7d5283a6cc0d1, which modifies copy.Image() to
add the new image's manifest to the values that it returns.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
This commit is contained in:
Nalin Dahyabhai 2018-10-11 14:55:07 -04:00
parent 7815a5801e
commit 2b97124e4a
10 changed files with 104 additions and 74 deletions

View File

@ -53,7 +53,7 @@ are also available
This library, by default, also depends on the GpgME and libostree C libraries. Either install them:
```sh
Fedora$ dnf install gpgme-devel libassuan-devel libostree-devel
Fedora$ dnf install gpgme-devel libassuan-devel ostree-devel
macOS$ brew install gpgme
```
or use the build tags described below to avoid the dependencies (e.g. using `go build -tags …`)

View File

@ -102,8 +102,9 @@ type Options struct {
}
// Image copies image from srcRef to destRef, using policyContext to validate
// source image admissibility.
func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (retErr error) {
// source image admissibility. It returns the manifest which was written to
// the new copy of the image.
func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (manifest []byte, retErr error) {
// NOTE this function uses an output parameter for the error return value.
// Setting this and returning is the ideal way to return an error.
//
@ -121,7 +122,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
dest, err := destRef.NewImageDestination(ctx, options.DestinationCtx)
if err != nil {
return errors.Wrapf(err, "Error initializing destination %s", transports.ImageName(destRef))
return nil, errors.Wrapf(err, "Error initializing destination %s", transports.ImageName(destRef))
}
defer func() {
if err := dest.Close(); err != nil {
@ -131,7 +132,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
rawSource, err := srcRef.NewImageSource(ctx, options.SourceCtx)
if err != nil {
return errors.Wrapf(err, "Error initializing source %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "Error initializing source %s", transports.ImageName(srcRef))
}
defer func() {
if err := rawSource.Close(); err != nil {
@ -151,63 +152,63 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
unparsedToplevel := image.UnparsedInstance(rawSource, nil)
multiImage, err := isMultiImage(ctx, unparsedToplevel)
if err != nil {
return errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(srcRef))
}
if !multiImage {
// The simple case: Just copy a single image.
if err := c.copyOneImage(ctx, policyContext, options, unparsedToplevel); err != nil {
return err
if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel); err != nil {
return nil, err
}
} else {
// This is a manifest list. Choose a single image and copy it.
// FIXME: Copy to destinations which support manifest lists, one image at a time.
instanceDigest, err := image.ChooseManifestInstanceFromManifestList(ctx, options.SourceCtx, unparsedToplevel)
if err != nil {
return errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef))
}
logrus.Debugf("Source is a manifest list; copying (only) instance %s", instanceDigest)
unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
if err := c.copyOneImage(ctx, policyContext, options, unparsedInstance); err != nil {
return err
if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedInstance); err != nil {
return nil, err
}
}
if err := c.dest.Commit(ctx); err != nil {
return errors.Wrap(err, "Error committing the finished image")
return nil, errors.Wrap(err, "Error committing the finished image")
}
return nil
return manifest, nil
}
// Image copies a single (on-manifest-list) image unparsedImage, using policyContext to validate
// source image admissibility.
func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage) (retErr error) {
func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage) (manifest []byte, retErr error) {
// The caller is handling manifest lists; this could happen only if a manifest list contains a manifest list.
// Make sure we fail cleanly in such cases.
multiImage, err := isMultiImage(ctx, unparsedImage)
if err != nil {
// FIXME FIXME: How to name a reference for the sub-image?
return errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference()))
return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference()))
}
if multiImage {
return fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image")
return nil, fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image")
}
// Please keep this policy check BEFORE reading any other information about the image.
// (the multiImage check above only matches the MIME type, which we have received anyway.
// Actual parsing of anything should be deferred.)
if allowed, err := policyContext.IsRunningImageAllowed(ctx, unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so.
return errors.Wrap(err, "Source image rejected")
return nil, errors.Wrap(err, "Source image rejected")
}
src, err := image.FromUnparsedImage(ctx, options.SourceCtx, unparsedImage)
if err != nil {
return errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference()))
return nil, errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference()))
}
if err := checkImageDestinationForCurrentRuntimeOS(ctx, options.DestinationCtx, src, c.dest); err != nil {
return err
return nil, err
}
var sigs [][]byte
@ -217,14 +218,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
c.Printf("Getting image source signatures\n")
s, err := src.Signatures(ctx)
if err != nil {
return errors.Wrap(err, "Error reading signatures")
return nil, errors.Wrap(err, "Error reading signatures")
}
sigs = s
}
if len(sigs) != 0 {
c.Printf("Checking if image destination supports signatures\n")
if err := c.dest.SupportsSignatures(ctx); err != nil {
return errors.Wrap(err, "Can not copy signatures")
return nil, errors.Wrap(err, "Can not copy signatures")
}
}
@ -237,28 +238,28 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
}
if err := ic.updateEmbeddedDockerReference(); err != nil {
return err
return nil, err
}
// We compute preferredManifestMIMEType only to show it in error messages.
// Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed.
preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := ic.determineManifestConversion(ctx, c.dest.SupportedManifestMIMETypes(), options.ForceManifestMIMEType)
if err != nil {
return err
return nil, err
}
// If src.UpdatedImageNeedsLayerDiffIDs(ic.manifestUpdates) will be true, it needs to be true by the time we get here.
ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates)
if err := ic.copyLayers(ctx); err != nil {
return err
return nil, err
}
// With docker/distribution registries we do not know whether the registry accepts schema2 or schema1 only;
// and at least with the OpenShift registry "acceptschema2" option, there is no way to detect the support
// without actually trying to upload something and getting a types.ManifestTypeRejectedError.
// So, try the preferred manifest MIME type. If the process succeeds, fine…
manifest, err := ic.copyUpdatedConfigAndManifest(ctx)
manifest, err = ic.copyUpdatedConfigAndManifest(ctx)
if err != nil {
logrus.Debugf("Writing manifest using preferred type %s failed: %v", preferredManifestMIMEType, err)
// … if it fails, _and_ the failure is because the manifest is rejected, we may have other options.
@ -266,14 +267,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
// We dont have other options.
// In principle the code below would handle this as well, but the resulting error message is fairly ugly.
// Dont bother the user with MIME types if we have no choice.
return err
return nil, err
}
// If the original MIME type is acceptable, determineManifestConversion always uses it as preferredManifestMIMEType.
// So if we are here, we will definitely be trying to convert the manifest.
// With !ic.canModifyManifest, that would just be a string of repeated failures for the same reason,
// so lets bail out early and with a better error message.
if !ic.canModifyManifest {
return errors.Wrap(err, "Writing manifest failed (and converting it is not possible)")
return nil, errors.Wrap(err, "Writing manifest failed (and converting it is not possible)")
}
// errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil.
@ -294,24 +295,24 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
break
}
if errs != nil {
return fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", "))
return nil, fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", "))
}
}
if options.SignBy != "" {
newSig, err := c.createSignature(manifest, options.SignBy)
if err != nil {
return err
return nil, err
}
sigs = append(sigs, newSig)
}
c.Printf("Storing signatures\n")
if err := c.dest.PutSignatures(ctx, sigs); err != nil {
return errors.Wrap(err, "Error writing signatures")
return nil, errors.Wrap(err, "Error writing signatures")
}
return nil
return manifest, nil
}
// Printf writes a formatted string to c.reportWriter.

View File

@ -259,7 +259,7 @@ func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password
case http.StatusUnauthorized:
return ErrUnauthorizedForCredentials
default:
return errors.Errorf("error occured with status code %q", resp.StatusCode)
return errors.Errorf("error occured with status code %d (%s)", resp.StatusCode, http.StatusText(resp.StatusCode))
}
}
@ -329,7 +329,7 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
} else {
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logrus.Debugf("error getting search results from v1 endpoint %q, status code %d", registry, resp.StatusCode)
logrus.Debugf("error getting search results from v1 endpoint %q, status code %d (%s)", registry, resp.StatusCode, http.StatusText(resp.StatusCode))
} else {
if err := json.NewDecoder(resp.Body).Decode(v1Res); err != nil {
return nil, err
@ -346,7 +346,7 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
} else {
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logrus.Errorf("error getting search results from v2 endpoint %q, status code %d", registry, resp.StatusCode)
logrus.Errorf("error getting search results from v2 endpoint %q, status code %d (%s)", registry, resp.StatusCode, http.StatusText(resp.StatusCode))
} else {
if err := json.NewDecoder(resp.Body).Decode(v2Res); err != nil {
return nil, err
@ -495,7 +495,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope
case http.StatusOK:
break
default:
return nil, errors.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL)
return nil, errors.Errorf("unexpected http code: %d (%s), URL: %s", res.StatusCode, http.StatusText(res.StatusCode), authReq.URL)
}
tokenBlob, err := ioutil.ReadAll(res.Body)
if err != nil {
@ -516,13 +516,13 @@ func (c *dockerClient) detectProperties(ctx context.Context) error {
url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry)
resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth)
if err != nil {
logrus.Debugf("Ping %s err %#v", url, err)
logrus.Debugf("Ping %s err %s (%#v)", url, err.Error(), err)
return err
}
defer resp.Body.Close()
logrus.Debugf("Ping %s status %d", url, resp.StatusCode)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
return errors.Errorf("error pinging registry %s, response code %d", c.registry, resp.StatusCode)
return errors.Errorf("error pinging registry %s, response code %d (%s)", c.registry, resp.StatusCode, http.StatusText(resp.StatusCode))
}
c.challenges = parseAuthHeader(resp.Header)
c.scheme = scheme
@ -542,8 +542,8 @@ func (c *dockerClient) detectProperties(ctx context.Context) error {
pingV1 := func(scheme string) bool {
url := fmt.Sprintf(resolvedPingV1URL, scheme, c.registry)
resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth)
logrus.Debugf("Ping %s err %#v", url, err)
if err != nil {
logrus.Debugf("Ping %s err %s (%#v)", url, err.Error(), err)
return false
}
defer resp.Body.Close()

View File

@ -73,7 +73,7 @@ func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types.
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// print url also
return nil, errors.Errorf("Invalid status code returned when fetching tags list %d", res.StatusCode)
return nil, errors.Errorf("Invalid status code returned when fetching tags list %d (%s)", res.StatusCode, http.StatusText(res.StatusCode))
}
var tagsHolder struct {

View File

@ -207,7 +207,7 @@ func (d *dockerImageDestination) HasBlob(ctx context.Context, info types.BlobInf
logrus.Debugf("... not present")
return false, -1, nil
default:
return false, -1, errors.Errorf("failed to read from destination repository %s: %v", reference.Path(d.ref.ref), http.StatusText(res.StatusCode))
return false, -1, errors.Errorf("failed to read from destination repository %s: %d (%s)", reference.Path(d.ref.ref), res.StatusCode, http.StatusText(res.StatusCode))
}
}

View File

@ -140,7 +140,7 @@ func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string)
resp, err = s.c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth)
if err == nil {
if resp.StatusCode != http.StatusOK {
err = errors.Errorf("error fetching external blob from %q: %d", url, resp.StatusCode)
err = errors.Errorf("error fetching external blob from %q: %d (%s)", url, resp.StatusCode, http.StatusText(resp.StatusCode))
logrus.Debug(err)
continue
}
@ -175,7 +175,7 @@ func (s *dockerImageSource) GetBlob(ctx context.Context, info types.BlobInfo) (i
}
if res.StatusCode != http.StatusOK {
// print url also
return nil, 0, errors.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode)
return nil, 0, errors.Errorf("Invalid status code returned when fetching blob %d (%s)", res.StatusCode, http.StatusText(res.StatusCode))
}
return res.Body, getBlobSize(res), nil
}
@ -274,7 +274,7 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (
if res.StatusCode == http.StatusNotFound {
return nil, true, nil
} else if res.StatusCode != http.StatusOK {
return nil, false, errors.Errorf("Error reading signature from %s: status %d", url.String(), res.StatusCode)
return nil, false, errors.Errorf("Error reading signature from %s: status %d (%s)", url.String(), res.StatusCode, http.StatusText(res.StatusCode))
}
sig, err := ioutil.ReadAll(res.Body)
if err != nil {

View File

@ -127,7 +127,7 @@ func (c *openshiftClient) doRequest(ctx context.Context, method, path string, re
if statusValid {
return nil, errors.New(status.Message)
}
return nil, errors.Errorf("HTTP error: status code: %d, body: %s", res.StatusCode, string(body))
return nil, errors.Errorf("HTTP error: status code: %d (%s), body: %s", res.StatusCode, http.StatusText(res.StatusCode), string(body))
}
return body, nil

View File

@ -165,9 +165,12 @@ func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
var auths dockerConfigFile
raw, err := ioutil.ReadFile(path)
if os.IsNotExist(err) {
auths.AuthConfigs = map[string]dockerAuthConfig{}
return auths, nil
if err != nil {
if os.IsNotExist(err) {
auths.AuthConfigs = map[string]dockerAuthConfig{}
return auths, nil
}
return dockerConfigFile{}, err
}
if legacyFormat {

View File

@ -34,6 +34,19 @@ func SetupCertificates(dir string, tlsc *tls.Config) error {
for _, f := range fs {
fullPath := filepath.Join(dir, f.Name())
if strings.HasSuffix(f.Name(), ".crt") {
logrus.Debugf(" crt: %s", fullPath)
data, err := ioutil.ReadFile(fullPath)
if err != nil {
if os.IsNotExist(err) {
// Dangling symbolic link?
// Race with someone who deleted the
// file after we read the directory's
// list of contents?
logrus.Warnf("error reading certificate %q: %v", fullPath, err)
continue
}
return err
}
if tlsc.RootCAs == nil {
systemPool, err := tlsconfig.SystemCertPool()
if err != nil {
@ -41,11 +54,6 @@ func SetupCertificates(dir string, tlsc *tls.Config) error {
}
tlsc.RootCAs = systemPool
}
logrus.Debugf(" crt: %s", fullPath)
data, err := ioutil.ReadFile(fullPath)
if err != nil {
return err
}
tlsc.RootCAs.AppendCertsFromPEM(data)
}
if strings.HasSuffix(f.Name(), ".cert") {

View File

@ -313,6 +313,10 @@ func (s storageImageDestination) DesiredLayerCompression() types.LayerCompressio
return types.PreserveOriginal
}
func (s *storageImageDestination) computeNextBlobCacheFile() string {
return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1)))
}
// PutBlob stores a layer or data blob in our temporary directory, checking that any information
// in the blobinfo matches the incoming data.
func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, isConfig bool) (types.BlobInfo, error) {
@ -328,7 +332,7 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
}
}
diffID := digest.Canonical.Digester()
filename := filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1)))
filename := s.computeNextBlobCacheFile()
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return errorBlobInfo, errors.Wrapf(err, "error creating temporary file %q", filename)
@ -504,7 +508,6 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
continue
}
var diff io.ReadCloser
// Check if there's already a layer with the ID that we'd give to the result of applying
// this layer blob to its parent, if it has one, or the blob's hex value otherwise.
diffID, haveDiffID := s.blobDiffIDs[blob.Digest]
@ -533,19 +536,11 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
lastLayer = layer.ID
continue
}
// Check if we cached a file with that blobsum. If we didn't already have a layer with
// the blob's contents, we should have gotten a copy.
if filename, ok := s.filenames[blob.Digest]; ok {
// Use the file's contents to initialize the layer.
file, err2 := os.Open(filename)
if err2 != nil {
return errors.Wrapf(err2, "error opening file %q", filename)
}
defer file.Close()
diff = file
}
if diff == nil {
// Try to find a layer with contents matching that blobsum.
// Check if we previously cached a file with that blob's contents. If we didn't,
// then we need to read the desired contents from a layer.
filename, ok := s.filenames[blob.Digest]
if !ok {
// Try to find the layer with contents matching that blobsum.
layer := ""
layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(blob.Digest)
if err2 == nil && len(layers) > 0 {
@ -559,24 +554,47 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
if layer == "" {
return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest)
}
// Use the layer's contents to initialize the new layer.
// Read the layer's contents.
noCompression := archive.Uncompressed
diffOptions := &storage.DiffOptions{
Compression: &noCompression,
}
diff, err2 = s.imageRef.transport.store.Diff("", layer, diffOptions)
diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions)
if err2 != nil {
return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest)
}
defer diff.Close()
// Copy the layer diff to a file. Diff() takes a lock that it holds
// until the ReadCloser that it returns is closed, and PutLayer() wants
// the same lock, so the diff can't just be directly streamed from one
// to the other.
filename = s.computeNextBlobCacheFile()
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
diff.Close()
return errors.Wrapf(err, "error creating temporary file %q", filename)
}
// Copy the data to the file.
// TODO: This can take quite some time, and should ideally be cancellable using
// ctx.Done().
_, err = io.Copy(file, diff)
diff.Close()
file.Close()
if err != nil {
return errors.Wrapf(err, "error storing blob to file %q", filename)
}
// Make sure that we can find this file later, should we need the layer's
// contents again.
s.filenames[blob.Digest] = filename
}
if diff == nil {
// This shouldn't have happened.
return errors.Errorf("error applying blob %q: content not found", blob.Digest)
// Read the cached blob and use it as a diff.
file, err := os.Open(filename)
if err != nil {
return errors.Wrapf(err, "error opening file %q", filename)
}
defer file.Close()
// Build the new layer using the diff, regardless of where it came from.
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, diff)
layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file)
if err != nil && errors.Cause(err) != storage.ErrDuplicateID {
return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest)
}