github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-21 10:09:07 +00:00
Code Issues Projects Releases Wiki Activity

centralize all writing of the index.json to one place

Browse Source 
Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
Avi Deitcher 2025-06-26 17:37:27 +03:00
parent 3f54a80824
commit 0e3147e387
8 changed files with 69 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/cmd/linuxkit/cache/cacheindex.go vendored Normal file
View File

@ -0,0 +1,45 @@
// ALL writes to index.json at the root of the cache directory
// must be done through calls in this file. This is to ensure that it always does
// proper locking.
package cache
import (
"errors"
"fmt"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/match"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
log "github.com/sirupsen/logrus"
)
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
func (p *Provider) DescriptorWrite(image string, desc v1.Descriptor) error {
if image == "" {
return errors.New("cannot write descriptor without reference name")
}
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = image
log.Debugf("writing descriptor for image %s", image)
// do we update an existing one? Or create a new one?
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
}
return nil
}
// RemoveDescriptors removes all descriptors that match the provided matcher.
// It does so in a parallel-access-safe way
func (p *Provider) RemoveDescriptors(matcher match.Matcher) error {
// will be replaced with locking
return p.cache.RemoveDescriptors(matcher)
}

19
src/cmd/linuxkit/cache/pull.go vendored
View File

@ -9,8 +9,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
namepkg "github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/validate"
@ -182,11 +180,6 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
return fmt.Errorf("error getting manifest for trusted image %s: %v", name, err)
}
// use the original image name in the annotation
annotations := map[string]string{
imagespec.AnnotationRefName: fullname,
}
// first attempt as an index
ii, err := desc.ImageIndex()
if err == nil {
@ -195,7 +188,7 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
if err := p.cache.WriteIndex(ii); err != nil {
return fmt.Errorf("unable to write index: %v", err)
}
if err := p.DescriptorWrite(&v1ref, desc.Descriptor); err != nil {
if err := p.DescriptorWrite(v1ref.String(), desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
if withArchReferences {
@ -206,11 +199,10 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
for _, m := range im.Manifests {
if m.MediaType.IsImage() && m.Platform != nil && m.Platform.Architecture != unknown && m.Platform.OS != unknown {
archSpecific := fmt.Sprintf("%s-%s", ref.String(), m.Platform.Architecture)
archRef, err := reference.Parse(archSpecific)
if err != nil {
if _, err := reference.Parse(archSpecific); err != nil {
return fmt.Errorf("unable to parse arch-specific reference %s: %v", archSpecific, err)
}
if err := p.DescriptorWrite(&archRef, m); err != nil {
if err := p.DescriptorWrite(archSpecific, m); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
}
@ -224,9 +216,12 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
return fmt.Errorf("provided image is neither an image nor an index: %s", name)
}
log.Debugf("ImageWrite retrieved %s is image, saving", fullname)
if err = p.cache.ReplaceImage(im, match.Name(fullname), layout.WithAnnotations(annotations)); err != nil {
if err = p.cache.WriteImage(im); err != nil {
return fmt.Errorf("unable to save image to cache: %v", err)
}
if err = p.DescriptorWrite(fullname, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write updated descriptor to cache: %v", err)
}
}
return nil

7
src/cmd/linuxkit/cache/push.go vendored
View File

@ -12,7 +12,6 @@ import (
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
log "github.com/sirupsen/logrus"
)
@ -131,11 +130,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
// it might not have existed, so we can add it locally
// use the original image name in the annotation
desc := m.DeepCopy()
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = archTag
if err := p.cache.AppendDescriptor(*desc); err != nil {
if err := p.DescriptorWrite(archTag, *desc); err != nil {
return fmt.Errorf("error appending descriptor for %s to layout index: %v", archTag, err)
}
img, err = p.cache.Image(m.Digest)

2
src/cmd/linuxkit/cache/remove.go vendored
View File

@ -65,7 +65,7 @@ func (p *Provider) Remove(name string) error {
log.Warnf("unable to remove blob %s for %s: %v", blob, name, err)
}
}
return p.cache.RemoveDescriptors(match.Name(name))
return p.RemoveDescriptors(match.Name(name))
}
func blobsForImage(img v1.Image) ([]v1.Hash, error) {

66
src/cmd/linuxkit/cache/write.go vendored
View File

@ -14,7 +14,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
@ -77,11 +76,6 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
return fmt.Errorf("error getting manifest for image %s: %v", pullImageName, err)
}
// use the original image name in the annotation
annotations := map[string]string{
imagespec.AnnotationRefName: image,
}
// first attempt as an index
ii, err := desc.ImageIndex()
if err == nil {
@ -120,7 +114,7 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
if err := p.cache.WriteIndex(ii); err != nil {
return fmt.Errorf("unable to write index: %v", err)
}
if err := p.DescriptorWrite(ref, desc.Descriptor); err != nil {
if err := p.DescriptorWrite(ref.String(), desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
} else {
@ -131,8 +125,11 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
return fmt.Errorf("provided image is neither an image nor an index: %s", image)
}
log.Debugf("ImageWrite retrieved %s is image, saving", pullImageName)
if err = p.cache.ReplaceImage(im, match.Name(image), layout.WithAnnotations(annotations)); err != nil {
return fmt.Errorf("unable to save image to cache: %v", err)
if err := p.cache.WriteImage(im); err != nil {
return fmt.Errorf("error writing image %s to cache: %v", pullImageName, err)
}
if err := p.DescriptorWrite(image, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write image descriptor to cache: %v", err)
}
}
return nil
@ -208,20 +205,11 @@ func (p *Provider) ImageLoad(r io.Reader) ([]v1.Descriptor, error) {
// each of these is either an image or an index
// either way, it gets added directly to the linuxkit cache index.
for _, desc := range im.Manifests {
if imgName, ok := desc.Annotations[images.AnnotationImageName]; ok {
// remove the old descriptor, if it exists
if err := p.cache.RemoveDescriptors(match.Name(imgName)); err != nil {
return nil, fmt.Errorf("unable to remove old descriptors for %s: %v", imgName, err)
imgName, ok := desc.Annotations[images.AnnotationImageName]
if ok {
if err := p.DescriptorWrite(imgName, desc); err != nil {
return nil, fmt.Errorf("error writing descriptor for %s: %v", imgName, err)
}
// save the image name under our proper annotation
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = imgName
}
log.Debugf("appending descriptor %#v", desc)
if err := p.cache.AppendDescriptor(desc); err != nil {
return nil, fmt.Errorf("error appending descriptor to layout index: %v", err)
}
descs = append(descs, desc)
}
@ -366,9 +354,6 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
return fmt.Errorf("error writing new index to json: %v", err)
}
// finally update the descriptor in the root
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptor from index.json: %v", err)
}
desc := v1.Descriptor{
MediaType: types.OCIImageIndex,
Size: size,
@ -377,36 +362,7 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
imagespec.AnnotationRefName: image,
},
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor to index.json: %v", err)
}
return nil
}
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) error {
if ref == nil {
return errors.New("cannot write descriptor without reference name")
}
image := ref.String()
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = image
log.Debugf("writing descriptor for image %s", image)
// do we update an existing one? Or create a new one?
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
}
return nil
return p.DescriptorWrite(ref.String(), desc)
}
func (p *Provider) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {

2
src/cmd/linuxkit/pkglib/build.go
View File

@ -561,7 +561,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
if err != nil {
return err
}
if err := c.DescriptorWrite(&ref, *desc); err != nil {
if err := c.DescriptorWrite(fullRelTag, *desc); err != nil {
return err
}
if err := c.Push(fullRelTag, "", bo.manifest, true); err != nil {

3
src/cmd/linuxkit/pkglib/build_test.go
View File

@ -407,12 +407,11 @@ func (c *cacheMocker) Push(name, remoteName string, withManifest, override bool)
return nil
}
func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) error {
func (c *cacheMocker) DescriptorWrite(image string, desc v1.Descriptor) error {
if !c.enabledDescriptorWrite {
return errors.New("descriptor disabled")
}
var (
image = ref.String()
im = v1.IndexManifest{
MediaType: types.OCIImageIndex,
Manifests: []v1.Descriptor{desc},

2
src/cmd/linuxkit/spec/cache.go
View File

@ -37,7 +37,7 @@ type CacheProvider interface {
ImageLoad(r io.Reader) ([]v1.Descriptor, error)
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
DescriptorWrite(ref *reference.Spec, descriptors v1.Descriptor) error
DescriptorWrite(image string, descriptors v1.Descriptor) error
// Push an image along with a multi-arch index from local cache to remote registry.
// name is the name as referenced in the local cache, remoteName is the name to give it remotely.
// If remoteName is empty, it is the same as name.