Merge pull request #460 from giuseppe/revendor-containers-image

vendor: revendor containers/image
This commit is contained in:
Miloslav Trmač 2017-11-25 13:30:45 +01:00 committed by GitHub
commit 78b29a5c2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 492 additions and 42 deletions

View File

@ -4,6 +4,8 @@ package ostree
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@ -12,18 +14,27 @@ import (
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/containers/image/manifest"
"github.com/containers/image/types"
"github.com/containers/storage/pkg/archive"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/ostreedev/ostree-go/pkg/otbuiltin"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
import "C"
type blobToImport struct {
Size int64
Digest digest.Digest
@ -35,18 +46,24 @@ type descriptor struct {
Digest digest.Digest `json:"digest"`
}
type fsLayersSchema1 struct {
BlobSum digest.Digest `json:"blobSum"`
}
type manifestSchema struct {
ConfigDescriptor descriptor `json:"config"`
LayersDescriptors []descriptor `json:"layers"`
LayersDescriptors []descriptor `json:"layers"`
FSLayers []fsLayersSchema1 `json:"fsLayers"`
}
type ostreeImageDestination struct {
ref ostreeReference
manifest string
schema manifestSchema
tmpDirPath string
blobs map[string]*blobToImport
digest digest.Digest
ref ostreeReference
manifest string
schema manifestSchema
tmpDirPath string
blobs map[string]*blobToImport
digest digest.Digest
signaturesLen int
repo *C.struct_OstreeRepo
}
// newImageDestination returns an ImageDestination for writing to an existing ostree.
@ -55,7 +72,7 @@ func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDes
if err := ensureDirectoryExists(tmpDirPath); err != nil {
return nil, err
}
return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, ""}, nil
return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, "", 0, nil}, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
@ -66,6 +83,9 @@ func (d *ostreeImageDestination) Reference() types.ImageReference {
// Close removes resources associated with an initialized ImageDestination, if any.
func (d *ostreeImageDestination) Close() error {
if d.repo != nil {
C.g_object_unref(C.gpointer(d.repo))
}
return os.RemoveAll(d.tmpDirPath)
}
@ -174,6 +194,35 @@ func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch strin
return err
}
func generateTarSplitMetadata(output *bytes.Buffer, file string) error {
mfz := gzip.NewWriter(output)
defer mfz.Close()
metaPacker := storage.NewJSONPacker(mfz)
stream, err := os.OpenFile(file, os.O_RDONLY, 0)
if err != nil {
return err
}
defer stream.Close()
gzReader, err := gzip.NewReader(stream)
if err != nil {
return err
}
defer gzReader.Close()
its, err := asm.NewInputTarStream(gzReader, metaPacker, nil)
if err != nil {
return err
}
_, err = io.Copy(ioutil.Discard, its)
if err != nil {
return err
}
return nil
}
func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToImport) error {
ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root")
@ -185,6 +234,11 @@ func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToIm
os.RemoveAll(destinationPath)
}()
var tarSplitOutput bytes.Buffer
if err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath); err != nil {
return err
}
if os.Getuid() == 0 {
if err := archive.UntarPath(blob.BlobPath, destinationPath); err != nil {
return err
@ -202,28 +256,35 @@ func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToIm
return err
}
}
return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size),
fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))})
}
func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobToImport) error {
ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
destinationPath := filepath.Dir(blob.BlobPath)
return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)})
}
func (d *ostreeImageDestination) importConfig(blob *blobToImport) error {
ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
return exec.Command("ostree", "commit",
"--repo", d.ref.repo,
fmt.Sprintf("--add-metadata-string=docker.size=%d", blob.Size),
"--branch", ostreeBranch, filepath.Dir(blob.BlobPath)).Run()
}
func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {
branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())
output, err := exec.Command("ostree", "show", "--repo", d.ref.repo, "--print-metadata-key=docker.size", branch).CombinedOutput()
if err != nil {
if bytes.Index(output, []byte("not found")) >= 0 || bytes.Index(output, []byte("No such")) >= 0 {
return false, -1, nil
if d.repo == nil {
repo, err := openRepo(d.ref.repo)
if err != nil {
return false, 0, err
}
return false, -1, err
d.repo = repo
}
size, err := strconv.ParseInt(strings.Trim(string(output), "'\n"), 10, 64)
branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())
found, data, err := readMetadata(d.repo, branch, "docker.size")
if err != nil || !found {
return found, -1, err
}
size, err := strconv.ParseInt(data, 10, 64)
if err != nil {
return false, -1, err
}
@ -272,6 +333,7 @@ func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error {
return err
}
}
d.signaturesLen = len(signatures)
return nil
}
@ -286,24 +348,37 @@ func (d *ostreeImageDestination) Commit() error {
return err
}
for _, layer := range d.schema.LayersDescriptors {
hash := layer.Digest.Hex()
checkLayer := func(hash string) error {
blob := d.blobs[hash]
// if the blob is not present in d.blobs then it is already stored in OSTree,
// and we don't need to import it.
if blob == nil {
continue
return nil
}
err := d.importBlob(repo, blob)
if err != nil {
return err
}
delete(d.blobs, hash)
return nil
}
for _, layer := range d.schema.LayersDescriptors {
hash := layer.Digest.Hex()
if err = checkLayer(hash); err != nil {
return err
}
}
for _, layer := range d.schema.FSLayers {
hash := layer.BlobSum.Hex()
if err = checkLayer(hash); err != nil {
return err
}
}
hash := d.schema.ConfigDescriptor.Digest.Hex()
blob := d.blobs[hash]
if blob != nil {
err := d.importConfig(blob)
// Import the other blobs that are not layers
for _, blob := range d.blobs {
err := d.importConfig(repo, blob)
if err != nil {
return err
}
@ -311,7 +386,9 @@ func (d *ostreeImageDestination) Commit() error {
manifestPath := filepath.Join(d.tmpDirPath, "manifest")
metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("docker.digest=%s", string(d.digest))}
metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)),
fmt.Sprintf("signatures=%d", d.signaturesLen),
fmt.Sprintf("docker.digest=%s", string(d.digest))}
err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata)
_, err = repo.CommitTransaction()

349
vendor/github.com/containers/image/ostree/ostree_src.go generated vendored Normal file
View File

@ -0,0 +1,349 @@
// +build !containers_image_ostree_stub
package ostree
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"strconv"
"strings"
"unsafe"
"github.com/containers/image/manifest"
"github.com/containers/image/types"
"github.com/containers/storage/pkg/ioutils"
"github.com/opencontainers/go-digest"
glib "github.com/ostreedev/ostree-go/pkg/glibobject"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
import "C"
type ostreeImageSource struct {
ref ostreeReference
tmpDir string
repo *C.struct_OstreeRepo
}
// newImageSource returns an ImageSource for reading from an existing directory.
func newImageSource(ctx *types.SystemContext, tmpDir string, ref ostreeReference) (types.ImageSource, error) {
return &ostreeImageSource{ref: ref, tmpDir: tmpDir}, nil
}
// Reference returns the reference used to set up this source.
func (s *ostreeImageSource) Reference() types.ImageReference {
return s.ref
}
// Close removes resources associated with an initialized ImageSource, if any.
func (s *ostreeImageSource) Close() error {
if s.repo != nil {
C.g_object_unref(C.gpointer(s.repo))
}
return nil
}
func (s *ostreeImageSource) getLayerSize(blob string) (int64, error) {
b := fmt.Sprintf("ociimage/%s", blob)
found, data, err := readMetadata(s.repo, b, "docker.size")
if err != nil || !found {
return 0, err
}
return strconv.ParseInt(data, 10, 64)
}
func (s *ostreeImageSource) getLenSignatures() (int64, error) {
b := fmt.Sprintf("ociimage/%s", s.ref.branchName)
found, data, err := readMetadata(s.repo, b, "signatures")
if err != nil {
return -1, err
}
if !found {
// if 'signatures' is not present, just return 0 signatures.
return 0, nil
}
return strconv.ParseInt(data, 10, 64)
}
func (s *ostreeImageSource) getTarSplitData(blob string) ([]byte, error) {
b := fmt.Sprintf("ociimage/%s", blob)
found, out, err := readMetadata(s.repo, b, "tarsplit.output")
if err != nil || !found {
return nil, err
}
return base64.StdEncoding.DecodeString(out)
}
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
func (s *ostreeImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
return nil, "", errors.Errorf(`Manifest lists are not supported by "ostree:"`)
}
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, "", err
}
s.repo = repo
}
b := fmt.Sprintf("ociimage/%s", s.ref.branchName)
found, out, err := readMetadata(s.repo, b, "docker.manifest")
if err != nil {
return nil, "", err
}
if !found {
return nil, "", errors.New("manifest not found")
}
m := []byte(out)
return m, manifest.GuessMIMEType(m), nil
}
func (s *ostreeImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
return nil, "", errors.New("manifest lists are not supported by this transport")
}
func openRepo(path string) (*C.struct_OstreeRepo, error) {
var cerr *C.GError
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
pathc := C.g_file_new_for_path(cpath)
defer C.g_object_unref(C.gpointer(pathc))
repo := C.ostree_repo_new(pathc)
r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr)))
if !r {
C.g_object_unref(C.gpointer(repo))
return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
return repo, nil
}
type ostreePathFileGetter struct {
repo *C.struct_OstreeRepo
parentRoot *C.GFile
}
type ostreeReader struct {
stream *C.GFileInputStream
}
func (o ostreeReader) Close() error {
C.g_object_unref(C.gpointer(o.stream))
return nil
}
func (o ostreeReader) Read(p []byte) (int, error) {
var cerr *C.GError
instanceCast := C.g_type_check_instance_cast((*C.GTypeInstance)(unsafe.Pointer(o.stream)), C.g_input_stream_get_type())
stream := (*C.GInputStream)(unsafe.Pointer(instanceCast))
b := C.g_input_stream_read_bytes(stream, (C.gsize)(cap(p)), nil, &cerr)
if b == nil {
return 0, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
defer C.g_bytes_unref(b)
count := int(C.g_bytes_get_size(b))
if count == 0 {
return 0, io.EOF
}
data := (*[1 << 30]byte)(unsafe.Pointer(C.g_bytes_get_data(b, nil)))[:count:count]
copy(p, data)
return count, nil
}
func readMetadata(repo *C.struct_OstreeRepo, commit, key string) (bool, string, error) {
var cerr *C.GError
var ref *C.char
defer C.free(unsafe.Pointer(ref))
cCommit := C.CString(commit)
defer C.free(unsafe.Pointer(cCommit))
if !glib.GoBool(glib.GBoolean(C.ostree_repo_resolve_rev(repo, cCommit, C.gboolean(1), &ref, &cerr))) {
return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
if ref == nil {
return false, "", nil
}
var variant *C.GVariant
if !glib.GoBool(glib.GBoolean(C.ostree_repo_load_variant(repo, C.OSTREE_OBJECT_TYPE_COMMIT, ref, &variant, &cerr))) {
return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
defer C.g_variant_unref(variant)
if variant != nil {
cKey := C.CString(key)
defer C.free(unsafe.Pointer(cKey))
metadata := C.g_variant_get_child_value(variant, 0)
defer C.g_variant_unref(metadata)
data := C.g_variant_lookup_value(metadata, (*C.gchar)(cKey), nil)
if data != nil {
defer C.g_variant_unref(data)
ptr := (*C.char)(C.g_variant_get_string(data, nil))
val := C.GoString(ptr)
return true, val, nil
}
}
return false, "", nil
}
func newOSTreePathFileGetter(repo *C.struct_OstreeRepo, commit string) (*ostreePathFileGetter, error) {
var cerr *C.GError
var parentRoot *C.GFile
cCommit := C.CString(commit)
defer C.free(unsafe.Pointer(cCommit))
if !glib.GoBool(glib.GBoolean(C.ostree_repo_read_commit(repo, cCommit, &parentRoot, nil, nil, &cerr))) {
return &ostreePathFileGetter{}, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
C.g_object_ref(C.gpointer(repo))
return &ostreePathFileGetter{repo: repo, parentRoot: parentRoot}, nil
}
func (o ostreePathFileGetter) Get(filename string) (io.ReadCloser, error) {
var file *C.GFile
if strings.HasPrefix(filename, "./") {
filename = filename[2:]
}
cfilename := C.CString(filename)
defer C.free(unsafe.Pointer(cfilename))
file = (*C.GFile)(C.g_file_resolve_relative_path(o.parentRoot, cfilename))
var cerr *C.GError
stream := C.g_file_read(file, nil, &cerr)
if stream == nil {
return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
}
return &ostreeReader{stream: stream}, nil
}
func (o ostreePathFileGetter) Close() {
C.g_object_unref(C.gpointer(o.repo))
C.g_object_unref(C.gpointer(o.parentRoot))
}
func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser, error) {
getter, err := newOSTreePathFileGetter(s.repo, commit)
if err != nil {
return nil, err
}
defer getter.Close()
return getter.Get(path)
}
// GetBlob returns a stream for the specified blob, and the blob's size.
func (s *ostreeImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
blob := info.Digest.Hex()
branch := fmt.Sprintf("ociimage/%s", blob)
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, 0, err
}
s.repo = repo
}
layerSize, err := s.getLayerSize(blob)
if err != nil {
return nil, 0, err
}
tarsplit, err := s.getTarSplitData(blob)
if err != nil {
return nil, 0, err
}
// if tarsplit is nil we are looking at the manifest. Return directly the file in /content
if tarsplit == nil {
file, err := s.readSingleFile(branch, "/content")
if err != nil {
return nil, 0, err
}
return file, layerSize, nil
}
mf := bytes.NewReader(tarsplit)
mfz, err := gzip.NewReader(mf)
if err != nil {
return nil, 0, err
}
defer mfz.Close()
metaUnpacker := storage.NewJSONUnpacker(mfz)
getter, err := newOSTreePathFileGetter(s.repo, branch)
if err != nil {
return nil, 0, err
}
ots := asm.NewOutputTarStream(getter, metaUnpacker)
pipeReader, pipeWriter := io.Pipe()
go func() {
io.Copy(pipeWriter, ots)
pipeWriter.Close()
}()
rc := ioutils.NewReadCloserWrapper(pipeReader, func() error {
getter.Close()
return ots.Close()
})
return rc, layerSize, nil
}
func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
if instanceDigest != nil {
return nil, errors.New("manifest lists are not supported by this transport")
}
lenSignatures, err := s.getLenSignatures()
if err != nil {
return nil, err
}
branch := fmt.Sprintf("ociimage/%s", s.ref.branchName)
if s.repo == nil {
repo, err := openRepo(s.ref.repo)
if err != nil {
return nil, err
}
s.repo = repo
}
signatures := [][]byte{}
for i := int64(1); i <= lenSignatures; i++ {
sigReader, err := s.readSingleFile(branch, fmt.Sprintf("/signature-%d", i))
if err != nil {
return nil, err
}
defer sigReader.Close()
sig, err := ioutil.ReadAll(sigReader)
if err != nil {
return nil, err
}
signatures = append(signatures, sig)
}
return signatures, nil
}

View File

@ -10,12 +10,12 @@ import (
"regexp"
"strings"
"github.com/pkg/errors"
"github.com/containers/image/directory/explicitfilepath"
"github.com/containers/image/docker/reference"
"github.com/containers/image/image"
"github.com/containers/image/transports"
"github.com/containers/image/types"
"github.com/pkg/errors"
)
const defaultOSTreeRepo = "/ostree/repo"
@ -66,6 +66,11 @@ type ostreeReference struct {
repo string
}
type ostreeImageCloser struct {
types.ImageCloser
size int64
}
func (t ostreeTransport) ParseReference(ref string) (types.ImageReference, error) {
var repo = ""
var image = ""
@ -110,7 +115,7 @@ func NewReference(image string, repo string) (types.ImageReference, error) {
// This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces
// from being ambiguous with values of PolicyConfigurationIdentity.
if strings.Contains(resolved, ":") {
return nil, errors.Errorf("Invalid OSTreeCI reference %s@%s: path %s contains a colon", image, repo, resolved)
return nil, errors.Errorf("Invalid OSTree reference %s@%s: path %s contains a colon", image, repo, resolved)
}
return ostreeReference{
@ -168,19 +173,38 @@ func (ref ostreeReference) PolicyConfigurationNamespaces() []string {
return res
}
func (s *ostreeImageCloser) Size() (int64, error) {
return s.size, nil
}
// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport.
// The caller must call .Close() on the returned ImageCloser.
// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
func (ref ostreeReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) {
return nil, errors.New("Reading ostree: images is currently not supported")
var tmpDir string
if ctx == nil || ctx.OSTreeTmpDirPath == "" {
tmpDir = os.TempDir()
} else {
tmpDir = ctx.OSTreeTmpDirPath
}
src, err := newImageSource(ctx, tmpDir, ref)
if err != nil {
return nil, err
}
return image.FromSource(ctx, src)
}
// NewImageSource returns a types.ImageSource for this reference.
// The caller must call .Close() on the returned ImageSource.
func (ref ostreeReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
return nil, errors.New("Reading ostree: images is currently not supported")
var tmpDir string
if ctx == nil || ctx.OSTreeTmpDirPath == "" {
tmpDir = os.TempDir()
} else {
tmpDir = ctx.OSTreeTmpDirPath
}
return newImageSource(ctx, tmpDir, ref)
}
// NewImageDestination returns a types.ImageDestination for this reference.