github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-19 01:06:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4084 from deitch/cache-platform-instead-of-arch

Browse Source 
internal restructure to use explicit platform instead of implicit arch in cache
This commit is contained in:
Avi Deitcher 2024-10-01 15:14:21 +03:00 committed by GitHub
commit e4d41061b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 682 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cmd/linuxkit/cache/find.go vendored
View File

@ -8,6 +8,7 @@ import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/partial"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// matchPlatformsOSArch because match.Platforms rejects it if the provided
@ -46,7 +47,7 @@ func matchAllAnnotations(annotations map[string]string) match.Matcher {
}
}
func (p *Provider) findImage(imageName, architecture string) (v1.Image, error) {
func (p *Provider) findImage(imageName string, platform imagespec.Platform) (v1.Image, error) {
root, err := p.FindRoot(imageName)
if err != nil {
return nil, err
@ -58,7 +59,7 @@ func (p *Provider) findImage(imageName, architecture string) (v1.Image, error) {
ii, err := root.ImageIndex()
if err == nil {
// we have the index, get the manifest that represents the manifest for the desired architecture
platform := v1.Platform{OS: "linux", Architecture: architecture}
platform := v1.Platform{OS: platform.OS, Architecture: platform.Architecture}
images, err := partial.FindImages(ii, matchPlatformsOSArch(platform))
if err != nil || len(images) < 1 {
return nil, fmt.Errorf("error retrieving image %s for platform %v from cache: %v", imageName, platform, err)

168
src/cmd/linuxkit/cache/source.go → src/cmd/linuxkit/cache/imagesource.go vendored
View File

@ -10,7 +10,6 @@ import (
"github.com/containerd/containerd/reference"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/partial"
@ -34,7 +33,7 @@ const (
type ImageSource struct {
ref *reference.Spec
provider *Provider
architecture string
platform *imagespec.Platform
descriptor *v1.Descriptor
}
@ -45,11 +44,11 @@ type spdxStatement struct {
// NewSource return an ImageSource for a specific ref and architecture in the given
// cache directory.
func (p *Provider) NewSource(ref *reference.Spec, architecture string, descriptor *v1.Descriptor) lktspec.ImageSource {
func (p *Provider) NewSource(ref *reference.Spec, platform *imagespec.Platform, descriptor *v1.Descriptor) lktspec.ImageSource {
return ImageSource{
ref: ref,
provider: p,
architecture: architecture,
platform: platform,
descriptor: descriptor,
}
}
@ -58,7 +57,7 @@ func (p *Provider) NewSource(ref *reference.Spec, architecture string, descripto
// architecture, if necessary.
func (c ImageSource) Config() (imagespec.ImageConfig, error) {
imageName := c.ref.String()
image, err := c.provider.findImage(imageName, c.architecture)
image, err := c.provider.findImage(imageName, *c.platform)
if err != nil {
return imagespec.ImageConfig{}, err
}
@ -84,7 +83,7 @@ func (c ImageSource) TarReader() (io.ReadCloser, error) {
imageName := c.ref.String()
// get a reference to the image
image, err := c.provider.findImage(imageName, c.architecture)
image, err := c.provider.findImage(imageName, *c.platform)
if err != nil {
return nil, err
}
@ -104,7 +103,7 @@ func (c ImageSource) V1TarReader(overrideName string) (io.ReadCloser, error) {
return nil, fmt.Errorf("error parsing image name: %v", err)
}
// get a reference to the image
image, err := c.provider.findImage(imageName, c.architecture)
image, err := c.provider.findImage(imageName, *c.platform)
if err != nil {
return nil, err
}
@ -129,7 +128,7 @@ func (c ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
return nil, fmt.Errorf("error parsing image name: %v", err)
}
// get a reference to the image
image, err := c.provider.findImage(imageName, c.architecture)
image, err := c.provider.findImage(imageName, *c.platform)
if err != nil {
return nil, err
}
@ -139,160 +138,37 @@ func (c ImageSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
defer w.Close()
tw := tar.NewWriter(w)
defer tw.Close()
// layout file
layoutFileBytes := []byte(layoutFile)
if err := tw.WriteHeader(&tar.Header{
Name: "oci-layout",
Mode: 0644,
Size: int64(len(layoutFileBytes)),
Typeflag: tar.TypeReg,
}); err != nil {
_ = w.CloseWithError(err)
return
}
if _, err := tw.Write(layoutFileBytes); err != nil {
if err := writeLayoutHeader(tw); err != nil {
_ = w.CloseWithError(err)
return
}
// make blobs directory
if err := tw.WriteHeader(&tar.Header{
Name: "blobs/",
Mode: 0755,
Typeflag: tar.TypeDir,
}); err != nil {
if err := writeLayoutImage(tw, image); err != nil {
_ = w.CloseWithError(err)
return
}
// make blobs/sha256 directory
if err := tw.WriteHeader(&tar.Header{
Name: "blobs/sha256/",
Mode: 0755,
Typeflag: tar.TypeDir,
}); err != nil {
_ = w.CloseWithError(err)
return
}
// write config, each layer, manifest, saving the digest for each
config, err := image.RawConfigFile()
imageDigest, err := image.Digest()
if err != nil {
_ = w.CloseWithError(err)
return
}
configDigest, configSize, err := v1.SHA256(bytes.NewReader(config))
imageSize, err := image.Size()
if err != nil {
_ = w.CloseWithError(err)
return
}
if err := tw.WriteHeader(&tar.Header{
Name: fmt.Sprintf("blobs/sha256/%s", configDigest.Hex),
Mode: 0644,
Typeflag: tar.TypeReg,
Size: configSize,
}); err != nil {
_ = w.CloseWithError(err)
return
}
if _, err := tw.Write(config); err != nil {
_ = w.CloseWithError(err)
return
}
layers, err := image.Layers()
if err != nil {
_ = w.CloseWithError(err)
return
}
for _, layer := range layers {
blob, err := layer.Compressed()
if err != nil {
_ = w.CloseWithError(err)
return
}
defer blob.Close()
blobDigest, err := layer.Digest()
if err != nil {
_ = w.CloseWithError(err)
return
}
blobSize, err := layer.Size()
if err != nil {
_ = w.CloseWithError(err)
return
}
if err := tw.WriteHeader(&tar.Header{
Name: fmt.Sprintf("blobs/sha256/%s", blobDigest.Hex),
Mode: 0644,
Size: blobSize,
Typeflag: tar.TypeReg,
}); err != nil {
_ = w.CloseWithError(err)
return
}
if _, err := io.Copy(tw, blob); err != nil {
_ = w.CloseWithError(err)
return
}
}
// write the manifest
manifest, err := image.RawManifest()
if err != nil {
_ = w.CloseWithError(err)
return
}
manifestDigest, manifestSize, err := v1.SHA256(bytes.NewReader(manifest))
if err != nil {
_ = w.CloseWithError(err)
return
}
if err := tw.WriteHeader(&tar.Header{
Name: fmt.Sprintf("blobs/sha256/%s", manifestDigest.Hex),
Mode: 0644,
Size: int64(len(manifest)),
Typeflag: tar.TypeReg,
}); err != nil {
_ = w.CloseWithError(err)
return
}
if _, err := tw.Write(manifest); err != nil {
_ = w.CloseWithError(err)
return
}
// write the index file
desc := v1.Descriptor{
MediaType: types.OCIImageIndex,
Size: manifestSize,
Digest: manifestDigest,
Size: imageSize,
Digest: imageDigest,
Annotations: map[string]string{
imagespec.AnnotationRefName: refName.String(),
},
}
ii := empty.Index
index, err := ii.IndexManifest()
if err != nil {
_ = w.CloseWithError(err)
return
}
index.Manifests = append(index.Manifests, desc)
rawIndex, err := json.MarshalIndent(index, "", " ")
if err != nil {
_ = w.CloseWithError(err)
return
}
// write the index
if err := tw.WriteHeader(&tar.Header{
Name: "index.json",
Mode: 0644,
Size: int64(len(rawIndex)),
}); err != nil {
_ = w.CloseWithError(err)
return
}
if _, err := tw.Write(rawIndex); err != nil {
if err := writeLayoutIndex(tw, desc); err != nil {
_ = w.CloseWithError(err)
return
}
@ -314,15 +190,15 @@ func (c ImageSource) SBoMs() ([]io.ReadCloser, error) {
}
// get the digest of the manifest that represents our targeted architecture
descs, err := partial.FindManifests(index, matchPlatformsOSArch(v1.Platform{OS: "linux", Architecture: c.architecture}))
descs, err := partial.FindManifests(index, matchPlatformsOSArch(v1.Platform{OS: c.platform.OS, Architecture: c.platform.Architecture}))
if err != nil {
return nil, err
}
if len(descs) < 1 {
return nil, fmt.Errorf("no manifest found for %s arch %s", c.ref.String(), c.architecture)
return nil, fmt.Errorf("no manifest found for %s platform %s", c.ref.String(), c.platform)
}
if len(descs) > 1 {
return nil, fmt.Errorf("multiple manifests found for %s arch %s", c.ref.String(), c.architecture)
return nil, fmt.Errorf("multiple manifests found for %s platform %s", c.ref.String(), c.platform)
}
// get the digest of the manifest that represents our targeted architecture
desc := descs[0]
@ -336,7 +212,7 @@ func (c ImageSource) SBoMs() ([]io.ReadCloser, error) {
return nil, err
}
if len(descs) > 1 {
return nil, fmt.Errorf("multiple manifests found for %s arch %s", c.ref.String(), c.architecture)
return nil, fmt.Errorf("multiple manifests found for %s platform %s", c.ref.String(), c.platform)
}
if len(descs) < 1 {
return nil, nil
@ -348,10 +224,10 @@ func (c ImageSource) SBoMs() ([]io.ReadCloser, error) {
return nil, err
}
if len(images) < 1 {
return nil, fmt.Errorf("no attestation image found for %s arch %s, even though the manifest exists", c.ref.String(), c.architecture)
return nil, fmt.Errorf("no attestation image found for %s platform %s, even though the manifest exists", c.ref.String(), c.platform)
}
if len(images) > 1 {
return nil, fmt.Errorf("multiple attestation images found for %s arch %s", c.ref.String(), c.architecture)
return nil, fmt.Errorf("multiple attestation images found for %s platform %s", c.ref.String(), c.platform)
}
image := images[0]
manifest, err := image.Manifest()
@ -363,7 +239,7 @@ func (c ImageSource) SBoMs() ([]io.ReadCloser, error) {
return nil, err
}
if len(manifest.Layers) != len(layers) {
return nil, fmt.Errorf("manifest layers and image layers do not match for the attestation for %s arch %s", c.ref.String(), c.architecture)
return nil, fmt.Errorf("manifest layers and image layers do not match for the attestation for %s platform %s", c.ref.String(), c.platform)
}
var readers []io.ReadCloser
for i, layer := range manifest.Layers {

162
src/cmd/linuxkit/cache/indexsource.go vendored Normal file
View File

@ -0,0 +1,162 @@
package cache
import (
"archive/tar"
"bytes"
"fmt"
"io"
"github.com/containerd/containerd/reference"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// IndexSource a source for an image in the OCI distribution cache.
// Implements a spec.ImageSource.
type IndexSource struct {
ref *reference.Spec
provider *Provider
descriptor *v1.Descriptor
platforms []imagespec.Platform
}
// NewIndexSource return an IndexSource for a specific ref in the given
// cache directory.
func (p *Provider) NewIndexSource(ref *reference.Spec, descriptor *v1.Descriptor, platforms []imagespec.Platform) lktspec.IndexSource {
return IndexSource{
ref: ref,
provider: p,
descriptor: descriptor,
platforms: platforms,
}
}
// Config return the imagespec.ImageConfig for the given source. Resolves to the
// architecture, if necessary.
func (c IndexSource) Image(platform imagespec.Platform) (spec.ImageSource, error) {
imageName := c.ref.String()
index, err := c.provider.findIndex(imageName)
if err != nil {
return nil, err
}
manifests, err := index.IndexManifest()
if err != nil {
return nil, err
}
for _, manifest := range manifests.Manifests {
if manifest.Platform != nil && manifest.Platform.Architecture == platform.Architecture && manifest.Platform.OS == platform.OS {
return c.provider.NewSource(c.ref, &platform, &manifest), nil
}
}
return nil, fmt.Errorf("no manifest found for platform %q", platform)
}
// OCITarReader return an io.ReadCloser to read the image as a v1 tarball whose contents match OCI v1 layout spec
func (c IndexSource) OCITarReader(overrideName string) (io.ReadCloser, error) {
imageName := c.ref.String()
saveName := imageName
if overrideName != "" {
saveName = overrideName
}
refName, err := name.ParseReference(saveName)
if err != nil {
return nil, fmt.Errorf("error parsing image name: %v", err)
}
// get a reference to the image
index, err := c.provider.findIndex(c.ref.String())
if err != nil {
return nil, err
}
// convert the writer to a reader
r, w := io.Pipe()
go func() {
defer w.Close()
tw := tar.NewWriter(w)
defer tw.Close()
if err := writeLayoutHeader(tw); err != nil {
_ = w.CloseWithError(err)
return
}
manifests, err := index.IndexManifest()
if err != nil {
_ = w.CloseWithError(err)
return
}
// for each manifest, write the manifest blob, then go through each manifest and find the image for it
// and write its blobs
for _, manifest := range manifests.Manifests {
// if we restricted this image source to certain platforms, we should only write those
if len(c.platforms) > 0 {
found := false
for _, platform := range c.platforms {
if platform.Architecture == manifest.Platform.Architecture && platform.OS == manifest.Platform.OS &&
(platform.Variant == "" || platform.Variant == manifest.Platform.Variant) {
found = true
break
}
}
if !found {
continue
}
}
switch manifest.MediaType {
case types.OCIManifestSchema1, types.DockerManifestSchema2:
// this is an image manifest
image, err := index.Image(manifest.Digest)
if err != nil {
_ = w.CloseWithError(err)
return
}
if err := writeLayoutImage(tw, image); err != nil {
_ = w.CloseWithError(err)
return
}
}
}
// write the index directly as a blob
indexSize, err := index.Size()
if err != nil {
_ = w.CloseWithError(err)
return
}
indexDigest, err := index.Digest()
if err != nil {
_ = w.CloseWithError(err)
return
}
indexBytes, err := index.RawManifest()
if err != nil {
_ = w.CloseWithError(err)
return
}
if err := writeLayoutBlob(tw, indexDigest.Hex, indexSize, bytes.NewReader(indexBytes)); err != nil {
_ = w.CloseWithError(err)
return
}
desc := v1.Descriptor{
MediaType: types.OCIImageIndex,
Size: indexSize,
Digest: indexDigest,
Annotations: map[string]string{
imagespec.AnnotationRefName: refName.String(),
},
}
if err := writeLayoutIndex(tw, desc); err != nil {
_ = w.CloseWithError(err)
return
}
}()
return r, nil
}
// Descriptor return the descriptor of the index.
func (c IndexSource) Descriptor() *v1.Descriptor {
return c.descriptor
}

145
src/cmd/linuxkit/cache/layout.go vendored Normal file
View File

@ -0,0 +1,145 @@
package cache
import (
"archive/tar"
"bytes"
"encoding/json"
"fmt"
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
)
func writeLayoutHeader(tw *tar.Writer) error {
// layout file
layoutFileBytes := []byte(layoutFile)
if err := tw.WriteHeader(&tar.Header{
Name: "oci-layout",
Mode: 0644,
Size: int64(len(layoutFileBytes)),
Typeflag: tar.TypeReg,
}); err != nil {
return err
}
if _, err := tw.Write(layoutFileBytes); err != nil {
return err
}
// make blobs directory
if err := tw.WriteHeader(&tar.Header{
Name: "blobs/",
Mode: 0755,
Typeflag: tar.TypeDir,
}); err != nil {
return err
}
// make blobs/sha256 directory
if err := tw.WriteHeader(&tar.Header{
Name: "blobs/sha256/",
Mode: 0755,
Typeflag: tar.TypeDir,
}); err != nil {
return err
}
return nil
}
func writeLayoutImage(tw *tar.Writer, image v1.Image) error {
// write config, each layer, manifest, saving the digest for each
manifest, err := image.Manifest()
if err != nil {
return err
}
configDesc := manifest.Config
configBytes, err := image.RawConfigFile()
if err != nil {
return err
}
if err := writeLayoutBlob(tw, configDesc.Digest.Hex, configDesc.Size, bytes.NewReader(configBytes)); err != nil {
return err
}
layers, err := image.Layers()
if err != nil {
return err
}
for _, layer := range layers {
blob, err := layer.Compressed()
if err != nil {
return err
}
defer blob.Close()
blobDigest, err := layer.Digest()
if err != nil {
return err
}
blobSize, err := layer.Size()
if err != nil {
return err
}
if err := writeLayoutBlob(tw, blobDigest.Hex, blobSize, blob); err != nil {
return err
}
}
// write the manifest
manifestSize, err := image.Size()
if err != nil {
return err
}
manifestDigest, err := image.Digest()
if err != nil {
return err
}
manifestBytes, err := image.RawManifest()
if err != nil {
return err
}
if err := writeLayoutBlob(tw, manifestDigest.Hex, manifestSize, bytes.NewReader(manifestBytes)); err != nil {
return err
}
return nil
}
func writeLayoutBlob(tw *tar.Writer, digest string, size int64, blob io.Reader) error {
if err := tw.WriteHeader(&tar.Header{
Name: fmt.Sprintf("blobs/sha256/%s", digest),
Mode: 0644,
Size: size,
Typeflag: tar.TypeReg,
}); err != nil {
return err
}
if _, err := io.Copy(tw, blob); err != nil {
return err
}
return nil
}
func writeLayoutIndex(tw *tar.Writer, desc v1.Descriptor) error {
ii := empty.Index
index, err := ii.IndexManifest()
if err != nil {
return err
}
index.Manifests = append(index.Manifests, desc)
rawIndex, err := json.MarshalIndent(index, "", " ")
if err != nil {
return err
}
// write the index
if err := tw.WriteHeader(&tar.Header{
Name: "index.json",
Mode: 0644,
Size: int64(len(rawIndex)),
}); err != nil {
return err
}
if _, err := tw.Write(rawIndex); err != nil {
return err
}
return nil
}

33
src/cmd/linuxkit/cache/platform.go vendored Normal file
View File

@ -0,0 +1,33 @@
package cache
import (
"fmt"
"strings"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
func platformString(p imagespec.Platform) string {
parts := []string{p.OS, p.Architecture}
if p.Variant != "" {
parts = append(parts, p.Variant)
}
return strings.Join(parts, "/")
}
func platformMessageGenerator(platforms []imagespec.Platform) string {
var platformMessage string
switch {
case len(platforms) == 0:
platformMessage = "all platforms"
case len(platforms) == 1:
platformMessage = fmt.Sprintf("platform %s", platformString(platforms[0]))
default:
var platStrings []string
for _, p := range platforms {
platStrings = append(platStrings, platformString(p))
}
platformMessage = fmt.Sprintf("platforms %s", strings.Join(platStrings, ","))
}
return platformMessage
}

57
src/cmd/linuxkit/cache/pull.go vendored
View File

@ -3,6 +3,7 @@ package cache
import (
"errors"
"fmt"
"strings"
"github.com/containerd/containerd/reference"
"github.com/google/go-containerregistry/pkg/authn"
@ -30,12 +31,13 @@ const (
// architecture, and any manifests that have no architecture at all. It will ignore manifests
// for other architectures. If no architecture is provided, it will validate all manifests.
// It also calculates the hash of each component.
func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lktspec.ImageSource, error) {
func (p *Provider) ValidateImage(ref *reference.Spec, platforms []imagespec.Platform) (lktspec.ImageSource, error) {
var (
imageIndex v1.ImageIndex
image v1.Image
imageName = ref.String()
desc *v1.Descriptor
platformMessage = platformMessageGenerator(platforms)
)
// next try the local cache
root, err := p.FindRoot(imageName)
@ -71,7 +73,15 @@ func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lkts
if err != nil {
return ImageSource{}, fmt.Errorf("could not get index manifest: %w", err)
}
var architectures = make(map[string]bool)
var (
targetPlatforms = make(map[string]bool)
foundPlatforms = make(map[string]bool)
)
for _, plat := range platforms {
pString := platformString(plat)
targetPlatforms[pString] = false
foundPlatforms[pString] = false
}
// ignore only other architectures; manifest entries that have no architectures at all
// are going to be additional metadata, so we need to check them
for _, m := range im.Manifests {
@ -80,29 +90,50 @@ func (p *Provider) ValidateImage(ref *reference.Spec, architecture string) (lkts
return ImageSource{}, fmt.Errorf("invalid image: %w", err)
}
}
if architecture != "" && m.Platform.Architecture == architecture && m.Platform.OS == linux {
if err := validateManifestContents(imageIndex, m.Digest); err != nil {
return ImageSource{}, fmt.Errorf("invalid image: %w", err)
}
architectures[architecture] = true
// go through each target platform, and see if this one matched. If it did, mark the target as
for _, plat := range platforms {
if plat.Architecture == m.Platform.Architecture && plat.OS == m.Platform.OS &&
(plat.Variant == "" || plat.Variant == m.Platform.Variant) {
targetPlatforms[platformString(plat)] = true
break
}
}
if architecture == "" || architectures[architecture] {
}
if len(platforms) == 0 {
return p.NewSource(
ref,
architecture,
nil,
desc,
), nil
}
return ImageSource{}, fmt.Errorf("index for %s did not contain image for platform linux/%s", imageName, architecture)
// we have cycled through all of the manifests, let's check if we have all of the platforms
var missing []string
for plat, found := range targetPlatforms {
if !found {
missing = append(missing, plat)
}
}
if len(missing) == 0 {
return p.NewSource(
ref,
nil,
desc,
), nil
}
return ImageSource{}, fmt.Errorf("index for %s did not contain image for platforms %s", imageName, strings.Join(missing, ", "))
case image != nil:
if len(platforms) > 1 {
return ImageSource{}, fmt.Errorf("image %s is not a multi-arch image, but asked for %s", imageName, platformMessage)
}
// we found a local image, make sure it is up to date
if err := validate.Image(image); err != nil {
return ImageSource{}, fmt.Errorf("invalid image, %s", err)
}
return p.NewSource(
ref,
architecture,
&platforms[0],
desc,
), nil
}
@ -164,7 +195,7 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
if err := p.cache.WriteIndex(ii); err != nil {
return fmt.Errorf("unable to write index: %v", err)
}
if _, err := p.DescriptorWrite(&v1ref, desc.Descriptor); err != nil {
if err := p.DescriptorWrite(&v1ref, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
if withArchReferences {
@ -179,7 +210,7 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
if err != nil {
return fmt.Errorf("unable to parse arch-specific reference %s: %v", archSpecific, err)
}
if _, err := p.DescriptorWrite(&archRef, m); err != nil {
if err := p.DescriptorWrite(&archRef, m); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
}

121
src/cmd/linuxkit/cache/write.go vendored
View File

@ -19,7 +19,6 @@ import (
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/types"
lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
log "github.com/sirupsen/logrus"
@ -40,45 +39,42 @@ const (
// If you just want to check the status of a local ref, use ValidateImage.
// Note that ImagePull does try ValidateImage first, so if the image is already in the cache, it will not
// do any network activity at all.
func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform, alwaysPull bool) error {
imageName := util.ReferenceExpand(ref.String())
canonicalRef, err := reference.Parse(imageName)
if err != nil {
return ImageSource{}, fmt.Errorf("invalid image name %s: %v", imageName, err)
return fmt.Errorf("invalid image name %s: %v", imageName, err)
}
ref = &canonicalRef
image := ref.String()
pullImageName := image
platformMessage := platformMessageGenerator(platforms)
remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
if trustedRef != "" {
pullImageName = trustedRef
}
log.Debugf("ImagePull to cache %s trusted reference %s", image, pullImageName)
// unless alwaysPull is set to true, check locally first
if alwaysPull {
log.Debugf("Instructed always to pull, so pulling image %s arch %s", image, architecture)
log.Debugf("Instructed always to pull, so pulling image %s %s", image, platformMessage)
} else {
imgSrc, err := p.ValidateImage(ref, architecture)
imgSrc, err := p.ValidateImage(ref, platforms)
switch {
case err == nil && imgSrc != nil:
log.Debugf("Image %s arch %s found in local cache, not pulling", image, architecture)
return imgSrc, nil
log.Debugf("Image %s %s found in local cache, not pulling", image, platformMessage)
return nil
case err != nil && errors.Is(err, &noReferenceError{}):
log.Debugf("Image %s arch %s not found in local cache, pulling", image, architecture)
log.Debugf("Image %s %s not found in local cache, pulling", image, platformMessage)
default:
log.Debugf("Image %s arch %s incomplete or invalid in local cache, error %v, pulling", image, architecture, err)
log.Debugf("Image %s %s incomplete or invalid in local cache, error %v, pulling", image, platformMessage, err)
}
// there was an error, so try to pull
}
remoteRef, err := name.ParseReference(pullImageName)
if err != nil {
return ImageSource{}, fmt.Errorf("invalid image name %s: %v", pullImageName, err)
return fmt.Errorf("invalid image name %s: %v", pullImageName, err)
}
desc, err := remote.Get(remoteRef, remoteOptions...)
if err != nil {
return ImageSource{}, fmt.Errorf("error getting manifest for trusted image %s: %v", pullImageName, err)
return fmt.Errorf("error getting manifest for image %s: %v", pullImageName, err)
}
// use the original image name in the annotation
@ -89,46 +85,57 @@ func (p *Provider) ImagePull(ref *reference.Spec, trustedRef, architecture strin
// first attempt as an index
ii, err := desc.ImageIndex()
if err == nil {
log.Debugf("ImageWrite retrieved %s is index, saving, first checking if it contains target arch %s", pullImageName, architecture)
log.Debugf("ImageWrite retrieved %s is index, saving, first checking if it contains target %s", pullImageName, platformMessage)
im, err := ii.IndexManifest()
if err != nil {
return ImageSource{}, fmt.Errorf("unable to get IndexManifest: %v", err)
return fmt.Errorf("unable to get IndexManifest: %v", err)
}
// only useful if it contains our architecture
var foundArch bool
var foundPlatforms []*v1.Platform
for _, m := range im.Manifests {
if m.MediaType.IsImage() && m.Platform != nil && m.Platform.Architecture == architecture && m.Platform.OS == linux {
foundArch = true
if m.MediaType.IsImage() && m.Platform != nil {
foundPlatforms = append(foundPlatforms, m.Platform)
}
}
// now see if we have all of the platforms we need
var missing []string
for _, requiredPlatform := range platforms {
// we did not find it, so maybe one satisfies it
var matchedPlatform bool
for _, p := range foundPlatforms {
if p.OS == requiredPlatform.OS && p.Architecture == requiredPlatform.Architecture && (p.Variant == requiredPlatform.Variant || requiredPlatform.Variant == "") {
// this one satisfies it, so do not count it missing
matchedPlatform = true
break
}
}
if !foundArch {
return ImageSource{}, fmt.Errorf("index %s does not contain target architecture %s", pullImageName, architecture)
if !matchedPlatform {
missing = append(missing, platformString(requiredPlatform))
}
}
if len(missing) > 0 {
return fmt.Errorf("index %s does not contain target platforms %s", pullImageName, strings.Join(missing, ","))
}
if err := p.cache.WriteIndex(ii); err != nil {
return ImageSource{}, fmt.Errorf("unable to write index: %v", err)
return fmt.Errorf("unable to write index: %v", err)
}
if _, err := p.DescriptorWrite(ref, desc.Descriptor); err != nil {
return ImageSource{}, fmt.Errorf("unable to write index descriptor to cache: %v", err)
if err := p.DescriptorWrite(ref, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
} else {
var im v1.Image
// try an image
im, err = desc.Image()
if err != nil {
return ImageSource{}, fmt.Errorf("provided image is neither an image nor an index: %s", image)
return fmt.Errorf("provided image is neither an image nor an index: %s", image)
}
log.Debugf("ImageWrite retrieved %s is image, saving", pullImageName)
if err = p.cache.ReplaceImage(im, match.Name(image), layout.WithAnnotations(annotations)); err != nil {
return ImageSource{}, fmt.Errorf("unable to save image to cache: %v", err)
return fmt.Errorf("unable to save image to cache: %v", err)
}
}
return p.NewSource(
ref,
architecture,
&desc.Descriptor,
), nil
return nil
}
// ImageLoad takes an OCI format image tar stream and writes it locally. It should be
@ -226,27 +233,27 @@ func (p *Provider) ImageLoad(r io.Reader) ([]v1.Descriptor, error) {
// does not pull down any images; entirely assumes that the subjects of the manifests are present.
// If a reference to the provided already exists and it is an index, updates the manifests in the
// existing index.
func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) {
func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) error {
image := ref.String()
log.Debugf("writing an index for %s", image)
if len(descriptors) < 1 {
return ImageSource{}, errors.New("cannot create index without any manifests")
return errors.New("cannot create index without any manifests")
}
ii, err := p.cache.ImageIndex()
if err != nil {
return ImageSource{}, fmt.Errorf("unable to get root index: %v", err)
return fmt.Errorf("unable to get root index: %v", err)
}
images, err := partial.FindImages(ii, match.Name(image))
if err != nil {
return ImageSource{}, fmt.Errorf("error parsing index: %v", err)
return fmt.Errorf("error parsing index: %v", err)
}
if err == nil && len(images) > 0 {
return ImageSource{}, fmt.Errorf("image named %s already exists in cache and is not an index", image)
return fmt.Errorf("image named %s already exists in cache and is not an index", image)
}
indexes, err := partial.FindIndexes(ii, match.Name(image))
if err != nil {
return ImageSource{}, fmt.Errorf("error parsing index: %v", err)
return fmt.Errorf("error parsing index: %v", err)
}
var im v1.IndexManifest
// do we update an existing one? Or create a new one?
@ -254,11 +261,11 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
// we already had one, so update just the referenced index and return
manifest, err := indexes[0].IndexManifest()
if err != nil {
return ImageSource{}, fmt.Errorf("unable to convert index for %s into its manifest: %v", image, err)
return fmt.Errorf("unable to convert index for %s into its manifest: %v", image, err)
}
oldhash, err := indexes[0].Digest()
if err != nil {
return ImageSource{}, fmt.Errorf("unable to get hash of existing index: %v", err)
return fmt.Errorf("unable to get hash of existing index: %v", err)
}
// we only care about avoiding duplicate arch/OS/Variant
var (
@ -335,7 +342,7 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
im = *manifest
// remove the old index
if err := p.cache.RemoveBlob(oldhash); err != nil {
return ImageSource{}, fmt.Errorf("unable to remove old index file: %v", err)
return fmt.Errorf("unable to remove old index file: %v", err)
}
} else {
// we did not have one, so create an index, store it, update the root index.json, and return
@ -349,18 +356,18 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
// write the updated index, remove the old one
b, err := json.Marshal(im)
if err != nil {
return ImageSource{}, fmt.Errorf("unable to marshal new index to json: %v", err)
return fmt.Errorf("unable to marshal new index to json: %v", err)
}
hash, size, err := v1.SHA256(bytes.NewReader(b))
if err != nil {
return ImageSource{}, fmt.Errorf("error calculating hash of index json: %v", err)
return fmt.Errorf("error calculating hash of index json: %v", err)
}
if err := p.cache.WriteBlob(hash, io.NopCloser(bytes.NewReader(b))); err != nil {
return ImageSource{}, fmt.Errorf("error writing new index to json: %v", err)
return fmt.Errorf("error writing new index to json: %v", err)
}
// finally update the descriptor in the root
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return ImageSource{}, fmt.Errorf("unable to remove old descriptor from index.json: %v", err)
return fmt.Errorf("unable to remove old descriptor from index.json: %v", err)
}
desc := v1.Descriptor{
MediaType: types.OCIImageIndex,
@ -371,21 +378,17 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
},
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return ImageSource{}, fmt.Errorf("unable to append new descriptor to index.json: %v", err)
return fmt.Errorf("unable to append new descriptor to index.json: %v", err)
}
return p.NewSource(
ref,
"",
&desc,
), nil
return nil
}
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) (lktspec.ImageSource, error) {
func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) error {
if ref == nil {
return ImageSource{}, errors.New("cannot write descriptor without reference name")
return errors.New("cannot write descriptor without reference name")
}
image := ref.String()
if desc.Annotations == nil {
@ -396,22 +399,18 @@ func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) (lkt
// do we update an existing one? Or create a new one?
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return ImageSource{}, fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
return fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return ImageSource{}, fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
return fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
}
return p.NewSource(
ref,
"",
&desc,
), nil
return nil
}
func (p *Provider) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
img, err := p.findImage(ref.String(), architecture)
img, err := p.findImage(ref.String(), imagespec.Platform{OS: linux, Architecture: architecture})
if err != nil {
return false, err
}

18
src/cmd/linuxkit/cache_export.go
View File

@ -4,17 +4,20 @@ import (
"io"
"os"
"runtime"
"strings"
"github.com/containerd/containerd/reference"
v1 "github.com/google/go-containerregistry/pkg/v1"
cachepkg "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func cacheExportCmd() *cobra.Command {
var (
arch string
platform string
outputFile string
format string
tagName string
@ -42,7 +45,16 @@ func cacheExportCmd() *cobra.Command {
log.Fatalf("unable to find image named %s: %v", name, err)
}
src := p.NewSource(&ref, arch, desc)
plat, err := v1.ParsePlatform(platform)
if err != nil {
log.Fatalf("invalid platform %s: %v", platform, err)
}
platspec := imagespec.Platform{
Architecture: plat.Architecture,
OS: plat.OS,
Variant: plat.Variant,
}
src := p.NewSource(&ref, &platspec, desc)
var reader io.ReadCloser
switch format {
case "docker":
@ -88,7 +100,7 @@ func cacheExportCmd() *cobra.Command {
},
}
cmd.Flags().StringVar(&arch, "arch", runtime.GOARCH, "Architecture to resolve an index to an image, if the provided image name is an index")
cmd.Flags().StringVar(&platform, "platform", strings.Join([]string{"linux", runtime.GOARCH}, "/"), "Platform to resolve an index to an image, if the provided image name is an index")
cmd.Flags().StringVar(&outputFile, "outfile", "", "Path to file to save output, '-' for stdout")
cmd.Flags().StringVar(&format, "format", "oci", "export format, one of 'oci' (OCI tar), 'docker' (docker tar), 'filesystem'")
cmd.Flags().StringVar(&tagName, "name", "", "override the provided image name in the exported tar file; useful only for format=oci")

9
src/cmd/linuxkit/moby/build/build.go
View File

@ -16,6 +16,8 @@ import (
"strings"
"github.com/containerd/containerd/reference"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
// drop-in 100% compatible replacement and 17% faster than compress/gzip.
gzip "github.com/klauspost/pgzip"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby"
@ -84,6 +86,8 @@ func OutputTypes() []string {
return ts
}
// outputImage given an image and a section, such as onboot, onshutdown or services, lay it out with correct location
// config, etc. in the filesystem, so runc can use it.
func outputImage(image *moby.Image, section string, index int, prefix string, m moby.Moby, idMap map[string]uint32, dupMap map[string]string, iw *tar.Writer, opts BuildOpts) error {
log.Infof(" Create OCI config for %s", image.Image)
imageName := util.ReferenceExpand(image.Image)
@ -91,7 +95,7 @@ func outputImage(image *moby.Image, section string, index int, prefix string, m
if err != nil {
return fmt.Errorf("could not resolve references for image %s: %v", image.Image, err)
}
src, err := imagePull(&ref, opts.Pull, opts.CacheDir, opts.DockerCache, opts.Arch)
src, err := imageSource(&ref, opts.Pull, opts.CacheDir, opts.DockerCache, imagespec.Platform{OS: "linux", Architecture: opts.Arch})
if err != nil {
return fmt.Errorf("could not pull image %s: %v", image.Image, err)
}
@ -281,8 +285,9 @@ func Build(m moby.Moby, w io.Writer, opts BuildOpts) error {
// get volume tarball from container
if err := ImageTar(location, vol.ImageRef(), lowerPath, apkTar, resolvconfSymlink, opts); err != nil {
return fmt.Errorf("failed to build volume tarball from %s: %v", vol.Name, err)
return fmt.Errorf("failed to build volume filesystem tarball from %s: %v", vol.Name, err)
}
// make upper and merged dirs which will be used for mounting
// no need to make lower dir, as it is made automatically by ImageTar()
tmpPath := strings.TrimPrefix(tmpDir, "/") + "/"

84
src/cmd/linuxkit/moby/build/image.go
View File

@ -11,6 +11,7 @@ import (
"github.com/containerd/containerd/reference"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/runtime-spec/specs-go"
log "github.com/sirupsen/logrus"
)
@ -175,14 +176,15 @@ func tarPrefix(path, location, refName string, tw tarWriter) error {
return nil
}
// ImageTar takes a Docker image and outputs it to a tar stream
// ImageTar takes a Docker image and outputs it to a tar stream as a merged filesystem for a specific architecture
// defined in opts.
// location is where it is in the linuxkit.yaml file
func ImageTar(location string, ref *reference.Spec, prefix string, tw tarWriter, resolv string, opts BuildOpts) (e error) {
refName := "empty"
if ref != nil {
refName = ref.String()
}
log.Debugf("image tar: %s %s", refName, prefix)
log.Debugf("image filesystem tar: %s %s %s", refName, prefix, opts.Arch)
if prefix != "" && prefix[len(prefix)-1] != '/' {
return fmt.Errorf("prefix does not end with /: %s", prefix)
}
@ -197,9 +199,8 @@ func ImageTar(location string, ref *reference.Spec, prefix string, tw tarWriter,
return nil
}
// pullImage first checks in the cache, then pulls the image.
// If pull==true, then it always tries to pull from registry.
src, err := imagePull(ref, opts.Pull, opts.CacheDir, opts.DockerCache, opts.Arch)
// get a handle on the image, optionally from docker, pulling from registry if necessary.
src, err := imageSource(ref, opts.Pull, opts.CacheDir, opts.DockerCache, imagespec.Platform{OS: "linux", Architecture: opts.Arch})
if err != nil {
return fmt.Errorf("could not pull image %s: %v", ref, err)
}
@ -356,6 +357,79 @@ func ImageTar(location string, ref *reference.Spec, prefix string, tw tarWriter,
return nil
}
// ImageOCITar takes an OCI image and outputs it to a tar stream as a v1 layout format.
// Will include all architectures, or, if specific ones provided, then only those.
// location is where it is in the linuxkit.yaml file
func ImageOCITar(location string, ref *reference.Spec, prefix string, tw tarWriter, opts BuildOpts, platforms []imagespec.Platform) (e error) {
refName := "empty"
if ref != nil {
refName = ref.String()
}
log.Debugf("image v1 layout tar: %s %s %s", refName, prefix, opts.Arch)
if prefix != "" && prefix[len(prefix)-1] != '/' {
return fmt.Errorf("prefix does not end with /: %s", prefix)
}
err := tarPrefix(prefix, location, refName, tw)
if err != nil {
return err
}
// if the image is blank, we do not need to do any more
if ref == nil {
return fmt.Errorf("no image reference provided")
}
// indexSource first checks in the cache, then pulls the image.
// If pull==true, then it always tries to pull from registry.
src, err := indexSource(ref, opts.Pull, opts.CacheDir, platforms)
if err != nil {
return fmt.Errorf("could not pull image %s: %v", ref, err)
}
contents, err := src.OCITarReader("")
if err != nil {
return fmt.Errorf("could not unpack image %s: %v", ref, err)
}
defer contents.Close()
tr := tar.NewReader(contents)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
// force PAX format, since it allows for unlimited Name/Linkname
// and we move all files below prefix.
hdr.Format = tar.FormatPAX
// ensure we record the source of the file in the PAX header
if hdr.PAXRecords == nil {
hdr.PAXRecords = make(map[string]string)
}
hdr.PAXRecords[moby.PaxRecordLinuxkitSource] = ref.String()
hdr.PAXRecords[moby.PaxRecordLinuxkitLocation] = location
hdr.Name = prefix + hdr.Name
if hdr.Typeflag == tar.TypeLink {
// hard links are referenced by full path so need to be adjusted
hdr.Linkname = prefix + hdr.Linkname
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, tr)
if err != nil {
return err
}
}
return nil
}
// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix, location string, ref *reference.Spec, config []byte, runtime moby.Runtime, tw tarWriter, readonly bool, dupMap map[string]string, opts BuildOpts) error { // nolint: lll
// if read only, just unpack in rootfs/ but otherwise set up for overlay

56
src/cmd/linuxkit/moby/build/images.go
View File

@ -5,20 +5,23 @@ import (
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/docker"
lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// imagePull pull an image from the OCI registry to the cache.
// If the image root already is in the cache, use it, unless
// the option pull is set to true.
// if alwaysPull, then do not even bother reading locally
func imagePull(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCache bool, architecture string) (lktspec.ImageSource, error) {
// imageSource given an image ref, get a handle on the image so it can be used as a source for its configuration
// and layers. If the image root already is in the cache, use it.
// If not in cache, pull it down from the OCI registry.
// Optionally can look in docker image cache first, before falling back to linuxkit cache and OCI registry.
// Optionally can be told to alwaysPull, in which case it always pulls from the OCI registry.
// Always works for a single architecture, as we are referencing a specific image.
func imageSource(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCache bool, platform imagespec.Platform) (lktspec.ImageSource, error) {
// several possibilities:
// - alwaysPull: try to pull it down from the registry to linuxkit cache, then fail
// - !alwaysPull && dockerCache: try to read it from docker, then try linuxkit cache, then try to pull from registry, then fail
// - !alwaysPull && !dockerCache: try linuxkit cache, then try to pull from registry, then fail
// first, try docker, if that is available
if !alwaysPull && dockerCache {
if err := docker.HasImage(ref, architecture); err == nil {
if err := docker.HasImage(ref, platform.Architecture); err == nil {
return docker.NewSource(ref), nil
}
// docker is not required, so any error - image not available, no docker, whatever - just gets ignored
@ -31,5 +34,44 @@ func imagePull(ref *reference.Spec, alwaysPull bool, cacheDir string, dockerCach
}
// if we made it here, we either did not have the image, or it was incomplete
return c.ImagePull(ref, ref.String(), architecture, alwaysPull)
if err := c.ImagePull(ref, []imagespec.Platform{platform}, alwaysPull); err != nil {
return nil, err
}
desc, err := c.FindDescriptor(ref)
if err != nil {
return nil, err
}
return c.NewSource(
ref,
&platform,
desc,
), nil
}
// indexSource given an image ref, get a handle on the index so it can be used as a source for its underlying images.
// If the index root already is in the cache, use it.
// If not in cache, pull it down from the OCI registry.
// Optionally can look in docker image cache first, before falling back to linuxkit cache and OCI registry.
// Optionally can be told to alwaysPull, in which case it always pulls from the OCI registry.
// Can provide architectures to list which ones to limit, or leave empty for all available.
func indexSource(ref *reference.Spec, alwaysPull bool, cacheDir string, platforms []imagespec.Platform) (lktspec.IndexSource, error) {
// get a reference to the local cache; we either will find the ref there or will pull to it
c, err := cache.NewProvider(cacheDir)
if err != nil {
return nil, err
}
// if we made it here, we either did not have the image, or it was incomplete
if err := c.ImagePull(ref, platforms, alwaysPull); err != nil {
return nil, err
}
desc, err := c.FindDescriptor(ref)
if err != nil {
return nil, err
}
return c.NewIndexSource(
ref,
desc,
platforms,
), nil
}

2
src/cmd/linuxkit/moby/config.go
View File

@ -79,6 +79,8 @@ type Volume struct {
Name string `yaml:"name" json:"name"`
Image string `yaml:"image,omitempty" json:"image,omitempty"`
ReadOnly bool `yaml:"readonly,omitempty" json:"readonly,omitempty"`
Format string `yaml:"format,omitempty" json:"format,omitempty"`
Platforms []string `yaml:"platforms,omitempty" json:"platforms,omitempty"`
ref *reference.Spec
}

10
src/cmd/linuxkit/pkglib/build.go
View File

@ -327,7 +327,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
case bo.pull:
// need to pull the image from the registry, else build
fmt.Fprintf(writer, "%s %s not found in local cache, trying to pull\n", ref, platform.Architecture)
if _, err := c.ImagePull(&ref, "", platform.Architecture, false); err == nil {
if err := c.ImagePull(&ref, []imagespec.Platform{platform}, false); err == nil {
fmt.Fprintf(writer, "%s pulled\n", ref)
// successfully pulled, no need to build, continue with next platform
continue
@ -470,7 +470,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
// - potentially create a release, including push and load into docker
// create a multi-arch index
if _, err := c.IndexWrite(&ref, descs...); err != nil {
if err := c.IndexWrite(&ref, descs...); err != nil {
return err
}
}
@ -490,7 +490,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
if err != nil {
return err
}
cacheSource := c.NewSource(&ref, platform.Architecture, desc)
cacheSource := c.NewSource(&ref, &platform, desc)
reader, err := cacheSource.V1TarReader(fmt.Sprintf("%s-%s", p.FullTag(), platform.Architecture))
if err != nil {
return fmt.Errorf("unable to get reader from cache: %v", err)
@ -562,7 +562,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
if err != nil {
return err
}
if _, err := c.DescriptorWrite(&ref, *desc); err != nil {
if err := c.DescriptorWrite(&ref, *desc); err != nil {
return err
}
if err := c.Push(fullRelTag, "", bo.manifest, true); err != nil {
@ -617,7 +617,7 @@ func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c lktspec.CacheProvi
if err != nil {
return nil, fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err)
}
if _, err := c.ImagePull(&ref, "", arch, false); err == nil {
if err := c.ImagePull(&ref, []imagespec.Platform{{Architecture: arch, OS: "linux"}}, false); err == nil {
fmt.Fprintf(writer, "image already found %s for arch %s", ref, arch)
desc, err := c.FindDescriptor(&ref)
if err != nil {

39
src/cmd/linuxkit/pkglib/build_test.go
View File

@ -240,21 +240,24 @@ type cacheMocker struct {
hashes map[string][]byte
}
func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (lktspec.ImageSource, error) {
func (c *cacheMocker) ImagePull(ref *reference.Spec, platforms []imagespec.Platform, alwaysPull bool) error {
if !c.enableImagePull {
return nil, errors.New("ImagePull disabled")
return errors.New("ImagePull disabled")
}
// make some random data for a layer
b := make([]byte, 256)
_, _ = rand.Read(b)
descs, err := c.imageWriteStream(bytes.NewReader(b))
if err != nil {
return nil, err
return err
}
if len(descs) != 1 {
return nil, fmt.Errorf("expected 1 descriptor, got %d", len(descs))
return fmt.Errorf("expected 1 descriptor, got %d", len(descs))
}
return c.NewSource(ref, architecture, &descs[1]), nil
if len(platforms) != 1 {
return fmt.Errorf("cache does not support multiple platforms %s", platforms)
}
return nil
}
func (c *cacheMocker) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
@ -359,9 +362,9 @@ func (c *cacheMocker) imageWriteStream(r io.Reader) ([]registry.Descriptor, erro
return []registry.Descriptor{desc}, nil
}
func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) {
func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.Descriptor) error {
if !c.enableIndexWrite {
return nil, errors.New("disabled")
return errors.New("disabled")
}
image := ref.String()
im := registry.IndexManifest{
@ -373,11 +376,11 @@ func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.De
// write the updated index, remove the old one
b, err := json.Marshal(im)
if err != nil {
return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
return fmt.Errorf("unable to marshal new index to json: %v", err)
}
hash, size, err := registry.SHA256(bytes.NewReader(b))
if err != nil {
return nil, fmt.Errorf("error calculating hash of index json: %v", err)
return fmt.Errorf("error calculating hash of index json: %v", err)
}
c.assignHash(hash.String(), b)
desc := registry.Descriptor{
@ -390,7 +393,7 @@ func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.De
}
c.appendImage(image, desc)
return c.NewSource(ref, "", &desc), nil
return nil
}
func (c *cacheMocker) Push(name, remoteName string, withManifest, override bool) error {
if !c.enablePush {
@ -402,9 +405,9 @@ func (c *cacheMocker) Push(name, remoteName string, withManifest, override bool)
return nil
}
func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc registry.Descriptor) (lktspec.ImageSource, error) {
func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc registry.Descriptor) error {
if !c.enabledDescriptorWrite {
return nil, errors.New("descriptor disabled")
return errors.New("descriptor disabled")
}
var (
image = ref.String()
@ -417,11 +420,11 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc registry.Descrip
// write the updated index, remove the old one
b, err := json.Marshal(im)
if err != nil {
return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
return fmt.Errorf("unable to marshal new index to json: %v", err)
}
hash, size, err := registry.SHA256(bytes.NewReader(b))
if err != nil {
return nil, fmt.Errorf("error calculating hash of index json: %v", err)
return fmt.Errorf("error calculating hash of index json: %v", err)
}
c.assignHash(hash.String(), b)
root := registry.Descriptor{
@ -434,7 +437,7 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc registry.Descrip
}
c.appendImage(image, root)
return c.NewSource(ref, "", &root), nil
return nil
}
func (c *cacheMocker) FindDescriptor(ref *reference.Spec) (*registry.Descriptor, error) {
name := ref.String()
@ -443,8 +446,8 @@ func (c *cacheMocker) FindDescriptor(ref *reference.Spec) (*registry.Descriptor,
}
return nil, fmt.Errorf("not found %s", name)
}
func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *registry.Descriptor) lktspec.ImageSource {
return cacheMockerSource{c, ref, architecture, descriptor}
func (c *cacheMocker) NewSource(ref *reference.Spec, platform *imagespec.Platform, descriptor *registry.Descriptor) lktspec.ImageSource {
return cacheMockerSource{c, ref, platform, descriptor}
}
func (c *cacheMocker) assignHash(hash string, b []byte) {
if c.hashes == nil {
@ -475,7 +478,7 @@ func (c *cacheMocker) GetContent(hash v1.Hash) (io.ReadCloser, error) {
type cacheMockerSource struct {
c *cacheMocker
ref *reference.Spec
architecture string
platform *imagespec.Platform
descriptor *registry.Descriptor
}

9
src/cmd/linuxkit/spec/cache.go
View File

@ -6,6 +6,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/reference"
v1 "github.com/google/go-containerregistry/pkg/v1"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// CacheProvider interface for a provide of a cache.
@ -19,7 +20,7 @@ type CacheProvider interface {
// ImagePull takes an image name and pulls it from a registry to the cache. It should be
// efficient and only write missing blobs, based on their content hash. If the ref already
// exists in the cache, it should not pull anything, unless alwaysPull is set to true.
ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (ImageSource, error)
ImagePull(ref *reference.Spec, platform []imagespec.Platform, alwaysPull bool) error
// ImageInCache takes an image name and checks if it exists in the cache, including checking that the given
// architecture is complete. Like ImagePull, it should be efficient and only write missing blobs, based on
// their content hash.
@ -30,20 +31,20 @@ type CacheProvider interface {
// Cache implementation determines whether it should pull missing blobs from a remote registry.
// If the provided reference already exists and it is an index, updates the manifests in the
// existing index.
IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (ImageSource, error)
IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) error
// ImageLoad takes an OCI format image tar stream in the io.Reader and writes it to the cache. It should be
// efficient and only write missing blobs, based on their content hash.
ImageLoad(r io.Reader) ([]v1.Descriptor, error)
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
DescriptorWrite(ref *reference.Spec, descriptors v1.Descriptor) (ImageSource, error)
DescriptorWrite(ref *reference.Spec, descriptors v1.Descriptor) error
// Push an image along with a multi-arch index from local cache to remote registry.
// name is the name as referenced in the local cache, remoteName is the name to give it remotely.
// If remoteName is empty, it is the same as name.
// if withManifest defined will push a multi-arch manifest
Push(name, remoteName string, withManifest, override bool) error
// NewSource return an ImageSource for a specific ref and architecture in the cache.
NewSource(ref *reference.Spec, architecture string, descriptor *v1.Descriptor) ImageSource
NewSource(ref *reference.Spec, platform *imagespec.Platform, descriptor *v1.Descriptor) ImageSource
// GetContent returns an io.Reader to the provided content as is, given a specific digest. It is
// up to the caller to validate it.
GetContent(hash v1.Hash) (io.ReadCloser, error)

15
src/cmd/linuxkit/spec/image.go
View File

@ -10,12 +10,12 @@ import (
// ImageSource interface to an image. It can have its config read, and a its containers
// can be read via an io.ReadCloser tar stream.
type ImageSource interface {
// Descriptor get the v1.Descriptor of the image
Descriptor() *v1.Descriptor
// Config get the config for the image
Config() (imagespec.ImageConfig, error)
// TarReader get the flattened filesystem of the image as a tar stream
TarReader() (io.ReadCloser, error)
// Descriptor get the v1.Descriptor of the image
Descriptor() *v1.Descriptor
// V1TarReader get the image as v1 tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image.
V1TarReader(overrideName string) (io.ReadCloser, error)
// OCITarReader get the image as an OCI tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image.
@ -23,3 +23,14 @@ type ImageSource interface {
// SBoM get the sbom for the image, if any is available
SBoMs() ([]io.ReadCloser, error)
}
// IndexSource interface to an image. It can have its config read, and a its containers
// can be read via an io.ReadCloser tar stream.
type IndexSource interface {
// Descriptor get the v1.Descriptor of the index
Descriptor() *v1.Descriptor
// Image get image for a specific architecture
Image(platform imagespec.Platform) (ImageSource, error)
// OCITarReader get the image as an OCI tarball, also compatible with `docker load`. If name arg is not "", override name of image in tarfile from default of image.
OCITarReader(overrideName string) (io.ReadCloser, error)
}