Merge pull request #4040 from deitch/export-formats

add cache export format OCI

src/cmd/linuxkit/cache/source.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
 	"fmt"
@@ -9,10 +10,12 @@ import (
 	"github.com/containerd/containerd/reference"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	intoto "github.com/in-toto/in-toto-golang/in_toto"
 	lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
@@ -21,6 +24,9 @@ import (
 
 const (
 	inTotoJsonMediaType = "application/vnd.in-toto+json"
+	layoutFile          = `{
+		"imageLayoutVersion": "1.0.0"
+	}`
 )
 
 // ImageSource a source for an image in the OCI distribution cache.
@@ -111,6 +117,189 @@ func (c ImageSource) V1TarReader(overrideName string) (io.ReadCloser, error) {
 	return r, nil
 }
 
+// OCITarReader return an io.ReadCloser to read the image as a v1 tarball
+func (c ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
+	imageName := c.ref.String()
+	saveName := imageName
+	if overrideName != "" {
+		saveName = overrideName
+	}
+	refName, err := name.ParseReference(saveName)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing image name: %v", err)
+	}
+	// get a reference to the image
+	image, err := c.provider.findImage(imageName, c.architecture)
+	if err != nil {
+		return nil, err
+	}
+	// convert the writer to a reader
+	r, w := io.Pipe()
+	go func() {
+		defer w.Close()
+		tw := tar.NewWriter(w)
+		defer tw.Close()
+		// layout file
+		layoutFileBytes := []byte(layoutFile)
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     "oci-layout",
+			Mode:     0644,
+			Size:     int64(len(layoutFileBytes)),
+			Typeflag: tar.TypeReg,
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		if _, err := tw.Write(layoutFileBytes); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+
+		// make blobs directory
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     "blobs/",
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		// make blobs/sha256 directory
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     "blobs/sha256/",
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		// write config, each layer, manifest, saving the digest for each
+		config, err := image.RawConfigFile()
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		configDigest, configSize, err := v1.SHA256(bytes.NewReader(config))
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     fmt.Sprintf("blobs/sha256/%s", configDigest.Hex),
+			Mode:     0644,
+			Typeflag: tar.TypeReg,
+			Size:     configSize,
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		if _, err := tw.Write(config); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		layers, err := image.Layers()
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		for _, layer := range layers {
+			blob, err := layer.Compressed()
+			if err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+			defer blob.Close()
+			blobDigest, err := layer.Digest()
+			if err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+			blobSize, err := layer.Size()
+			if err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+			if err := tw.WriteHeader(&tar.Header{
+				Name:     fmt.Sprintf("blobs/sha256/%s", blobDigest.Hex),
+				Mode:     0644,
+				Size:     blobSize,
+				Typeflag: tar.TypeReg,
+			}); err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+			if _, err := io.Copy(tw, blob); err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+		}
+		// write the manifest
+		manifest, err := image.RawManifest()
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		manifestDigest, manifestSize, err := v1.SHA256(bytes.NewReader(manifest))
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     fmt.Sprintf("blobs/sha256/%s", manifestDigest.Hex),
+			Mode:     0644,
+			Size:     int64(len(manifest)),
+			Typeflag: tar.TypeReg,
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		if _, err := tw.Write(manifest); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		// write the index file
+		desc := v1.Descriptor{
+			MediaType: types.OCIImageIndex,
+			Size:      manifestSize,
+			Digest:    manifestDigest,
+			Annotations: map[string]string{
+				imagespec.AnnotationRefName: refName.String(),
+			},
+		}
+		ii := empty.Index
+
+		index, err := ii.IndexManifest()
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+
+		index.Manifests = append(index.Manifests, desc)
+
+		rawIndex, err := json.MarshalIndent(index, "", "   ")
+		if err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		// write the index
+		if err := tw.WriteHeader(&tar.Header{
+			Name: "index.json",
+			Mode: 0644,
+			Size: int64(len(rawIndex)),
+		}); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+		if _, err := tw.Write(rawIndex); err != nil {
+			_ = w.CloseWithError(err)
+			return
+		}
+	}()
+	return r, nil
+}
+
 // Descriptor return the descriptor of the image.
 func (c ImageSource) Descriptor() *v1.Descriptor {
 	return c.descriptor

src/cmd/linuxkit/cache_export.go

@@ -45,12 +45,18 @@ func cacheExportCmd() *cobra.Command {
 			src := p.NewSource(&ref, arch, desc)
 			var reader io.ReadCloser
 			switch format {
-			case "oci":
+			case "docker":
 				fullTagName := fullname
 				if tagName != "" {
 					fullTagName = util.ReferenceExpand(tagName)
 				}
 				reader, err = src.V1TarReader(fullTagName)
+			case "oci":
+				fullTagName := fullname
+				if tagName != "" {
+					fullTagName = util.ReferenceExpand(tagName)
+				}
+				reader, err = src.OCITarReader(fullTagName)
 			case "filesystem":
 				reader, err = src.TarReader()
 			default:
@@ -84,7 +90,7 @@ func cacheExportCmd() *cobra.Command {
 
 	cmd.Flags().StringVar(&arch, "arch", runtime.GOARCH, "Architecture to resolve an index to an image, if the provided image name is an index")
 	cmd.Flags().StringVar(&outputFile, "outfile", "", "Path to file to save output, '-' for stdout")
-	cmd.Flags().StringVar(&format, "format", "oci", "export format, one of 'oci', 'filesystem'")
+	cmd.Flags().StringVar(&format, "format", "oci", "export format, one of 'oci' (OCI tar), 'docker' (docker tar), 'filesystem'")
 	cmd.Flags().StringVar(&tagName, "name", "", "override the provided image name in the exported tar file; useful only for format=oci")
 
 	return cmd

src/cmd/linuxkit/docker/source.go

@@ -86,6 +86,11 @@ func (d ImageSource) V1TarReader(overrideName string) (io.ReadCloser, error) {
 	return Save(saveName)
 }
 
+// OCITarReader return an io.ReadCloser to read the save of the image
+func (d ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
+	return nil, fmt.Errorf("unsupported")
+}
+
 // Descriptor return the descriptor of the image.
 func (d ImageSource) Descriptor() *v1.Descriptor {
 	return nil

src/cmd/linuxkit/pkglib/build_test.go

@@ -494,6 +494,15 @@ func (c cacheMockerSource) V1TarReader(overrideName string) (io.ReadCloser, erro
 	_, _ = rand.Read(b)
 	return io.NopCloser(bytes.NewReader(b)), nil
 }
+func (c cacheMockerSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
+	_, found := c.c.images[c.ref.String()]
+	if !found {
+		return nil, fmt.Errorf("no image found with ref: %s", c.ref.String())
+	}
+	b := make([]byte, 256)
+	_, _ = rand.Read(b)
+	return io.NopCloser(bytes.NewReader(b)), nil
+}
 func (c cacheMockerSource) Descriptor() *registry.Descriptor {
 	return c.descriptor
 }

src/cmd/linuxkit/spec/image.go

@@ -18,6 +18,8 @@ type ImageSource interface {
 	Descriptor() *v1.Descriptor
 	// V1TarReader get the image as v1 tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image.
 	V1TarReader(overrideName string) (io.ReadCloser, error)
+	// OCITarReader get the image as an OCI tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image.
+	OCITarReader(overrideName string) (io.ReadCloser, error)
 	// SBoM get the sbom for the image, if any is available
 	SBoMs() ([]io.ReadCloser, error)
 }