From f5dcefc7c2c927f63f9857b76c768bf23d8ec33d Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Thu, 16 May 2024 14:16:18 +0300 Subject: [PATCH] add cache export format OCI Signed-off-by: Avi Deitcher --- src/cmd/linuxkit/cache/source.go | 189 ++++++++++++++++++++++++++ src/cmd/linuxkit/cache_export.go | 10 +- src/cmd/linuxkit/docker/source.go | 5 + src/cmd/linuxkit/pkglib/build_test.go | 9 ++ src/cmd/linuxkit/spec/image.go | 2 + 5 files changed, 213 insertions(+), 2 deletions(-) diff --git a/src/cmd/linuxkit/cache/source.go b/src/cmd/linuxkit/cache/source.go index 0a3fa60a9..878535571 100644 --- a/src/cmd/linuxkit/cache/source.go +++ b/src/cmd/linuxkit/cache/source.go @@ -1,6 +1,7 @@ package cache import ( + "archive/tar" "bytes" "encoding/json" "fmt" @@ -9,10 +10,12 @@ import ( "github.com/containerd/containerd/reference" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/match" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" intoto "github.com/in-toto/in-toto-golang/in_toto" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/util" @@ -21,6 +24,9 @@ import ( const ( inTotoJsonMediaType = "application/vnd.in-toto+json" + layoutFile = `{ + "imageLayoutVersion": "1.0.0" + }` ) // ImageSource a source for an image in the OCI distribution cache. @@ -111,6 +117,189 @@ func (c ImageSource) V1TarReader(overrideName string) (io.ReadCloser, error) { return r, nil } +// OCITarReader return an io.ReadCloser to read the image as a v1 tarball +func (c ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) { + imageName := c.ref.String() + saveName := imageName + if overrideName != "" { + saveName = overrideName + } + refName, err := name.ParseReference(saveName) + if err != nil { + return nil, fmt.Errorf("error parsing image name: %v", err) + } + // get a reference to the image + image, err := c.provider.findImage(imageName, c.architecture) + if err != nil { + return nil, err + } + // convert the writer to a reader + r, w := io.Pipe() + go func() { + defer w.Close() + tw := tar.NewWriter(w) + defer tw.Close() + // layout file + layoutFileBytes := []byte(layoutFile) + if err := tw.WriteHeader(&tar.Header{ + Name: "oci-layout", + Mode: 0644, + Size: int64(len(layoutFileBytes)), + Typeflag: tar.TypeReg, + }); err != nil { + _ = w.CloseWithError(err) + return + } + if _, err := tw.Write(layoutFileBytes); err != nil { + _ = w.CloseWithError(err) + return + } + + // make blobs directory + if err := tw.WriteHeader(&tar.Header{ + Name: "blobs/", + Mode: 0755, + Typeflag: tar.TypeDir, + }); err != nil { + _ = w.CloseWithError(err) + return + } + // make blobs/sha256 directory + if err := tw.WriteHeader(&tar.Header{ + Name: "blobs/sha256/", + Mode: 0755, + Typeflag: tar.TypeDir, + }); err != nil { + _ = w.CloseWithError(err) + return + } + // write config, each layer, manifest, saving the digest for each + config, err := image.RawConfigFile() + if err != nil { + _ = w.CloseWithError(err) + return + } + configDigest, configSize, err := v1.SHA256(bytes.NewReader(config)) + if err != nil { + _ = w.CloseWithError(err) + return + } + + if err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("blobs/sha256/%s", configDigest.Hex), + Mode: 0644, + Typeflag: tar.TypeReg, + Size: configSize, + }); err != nil { + _ = w.CloseWithError(err) + return + } + if _, err := tw.Write(config); err != nil { + _ = w.CloseWithError(err) + return + } + layers, err := image.Layers() + if err != nil { + _ = w.CloseWithError(err) + return + } + for _, layer := range layers { + blob, err := layer.Compressed() + if err != nil { + _ = w.CloseWithError(err) + return + } + defer blob.Close() + blobDigest, err := layer.Digest() + if err != nil { + _ = w.CloseWithError(err) + return + } + blobSize, err := layer.Size() + if err != nil { + _ = w.CloseWithError(err) + return + } + if err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("blobs/sha256/%s", blobDigest.Hex), + Mode: 0644, + Size: blobSize, + Typeflag: tar.TypeReg, + }); err != nil { + _ = w.CloseWithError(err) + return + } + if _, err := io.Copy(tw, blob); err != nil { + _ = w.CloseWithError(err) + return + } + } + // write the manifest + manifest, err := image.RawManifest() + if err != nil { + _ = w.CloseWithError(err) + return + } + manifestDigest, manifestSize, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + _ = w.CloseWithError(err) + return + } + + if err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("blobs/sha256/%s", manifestDigest.Hex), + Mode: 0644, + Size: int64(len(manifest)), + Typeflag: tar.TypeReg, + }); err != nil { + _ = w.CloseWithError(err) + return + } + if _, err := tw.Write(manifest); err != nil { + _ = w.CloseWithError(err) + return + } + // write the index file + desc := v1.Descriptor{ + MediaType: types.OCIImageIndex, + Size: manifestSize, + Digest: manifestDigest, + Annotations: map[string]string{ + imagespec.AnnotationRefName: refName.String(), + }, + } + ii := empty.Index + + index, err := ii.IndexManifest() + if err != nil { + _ = w.CloseWithError(err) + return + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + _ = w.CloseWithError(err) + return + } + // write the index + if err := tw.WriteHeader(&tar.Header{ + Name: "index.json", + Mode: 0644, + Size: int64(len(rawIndex)), + }); err != nil { + _ = w.CloseWithError(err) + return + } + if _, err := tw.Write(rawIndex); err != nil { + _ = w.CloseWithError(err) + return + } + }() + return r, nil +} + // Descriptor return the descriptor of the image. func (c ImageSource) Descriptor() *v1.Descriptor { return c.descriptor diff --git a/src/cmd/linuxkit/cache_export.go b/src/cmd/linuxkit/cache_export.go index 0aab01f06..469595272 100644 --- a/src/cmd/linuxkit/cache_export.go +++ b/src/cmd/linuxkit/cache_export.go @@ -45,12 +45,18 @@ func cacheExportCmd() *cobra.Command { src := p.NewSource(&ref, arch, desc) var reader io.ReadCloser switch format { - case "oci": + case "docker": fullTagName := fullname if tagName != "" { fullTagName = util.ReferenceExpand(tagName) } reader, err = src.V1TarReader(fullTagName) + case "oci": + fullTagName := fullname + if tagName != "" { + fullTagName = util.ReferenceExpand(tagName) + } + reader, err = src.OCITarReader(fullTagName) case "filesystem": reader, err = src.TarReader() default: @@ -84,7 +90,7 @@ func cacheExportCmd() *cobra.Command { cmd.Flags().StringVar(&arch, "arch", runtime.GOARCH, "Architecture to resolve an index to an image, if the provided image name is an index") cmd.Flags().StringVar(&outputFile, "outfile", "", "Path to file to save output, '-' for stdout") - cmd.Flags().StringVar(&format, "format", "oci", "export format, one of 'oci', 'filesystem'") + cmd.Flags().StringVar(&format, "format", "oci", "export format, one of 'oci' (OCI tar), 'docker' (docker tar), 'filesystem'") cmd.Flags().StringVar(&tagName, "name", "", "override the provided image name in the exported tar file; useful only for format=oci") return cmd diff --git a/src/cmd/linuxkit/docker/source.go b/src/cmd/linuxkit/docker/source.go index 755d0cd3e..4c408a1e7 100644 --- a/src/cmd/linuxkit/docker/source.go +++ b/src/cmd/linuxkit/docker/source.go @@ -86,6 +86,11 @@ func (d ImageSource) V1TarReader(overrideName string) (io.ReadCloser, error) { return Save(saveName) } +// OCITarReader return an io.ReadCloser to read the save of the image +func (d ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) { + return nil, fmt.Errorf("unsupported") +} + // Descriptor return the descriptor of the image. func (d ImageSource) Descriptor() *v1.Descriptor { return nil diff --git a/src/cmd/linuxkit/pkglib/build_test.go b/src/cmd/linuxkit/pkglib/build_test.go index f0ffa3cbf..e129bf4c0 100644 --- a/src/cmd/linuxkit/pkglib/build_test.go +++ b/src/cmd/linuxkit/pkglib/build_test.go @@ -494,6 +494,15 @@ func (c cacheMockerSource) V1TarReader(overrideName string) (io.ReadCloser, erro _, _ = rand.Read(b) return io.NopCloser(bytes.NewReader(b)), nil } +func (c cacheMockerSource) OCITarReader(overrideName string) (io.ReadCloser, error) { + _, found := c.c.images[c.ref.String()] + if !found { + return nil, fmt.Errorf("no image found with ref: %s", c.ref.String()) + } + b := make([]byte, 256) + _, _ = rand.Read(b) + return io.NopCloser(bytes.NewReader(b)), nil +} func (c cacheMockerSource) Descriptor() *registry.Descriptor { return c.descriptor } diff --git a/src/cmd/linuxkit/spec/image.go b/src/cmd/linuxkit/spec/image.go index c17d8513e..1ac3c3b41 100644 --- a/src/cmd/linuxkit/spec/image.go +++ b/src/cmd/linuxkit/spec/image.go @@ -18,6 +18,8 @@ type ImageSource interface { Descriptor() *v1.Descriptor // V1TarReader get the image as v1 tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image. V1TarReader(overrideName string) (io.ReadCloser, error) + // OCITarReader get the image as an OCI tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image. + OCITarReader(overrideName string) (io.ReadCloser, error) // SBoM get the sbom for the image, if any is available SBoMs() ([]io.ReadCloser, error) }