12 Commits

Author SHA1 Message Date
Avi Deitcher
bc44cb899c fix registry auth (#4146)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-07-15 14:07:20 +03:00
Avi Deitcher
33ee27971d for pkg build builder, copy config over rather than bind-mounting it (#4145)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-07-13 21:18:00 +03:00
Avi Deitcher
fa246722e2 add missing CRLF in log message (#4144)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-07-13 17:30:59 +03:00
Avi Deitcher
c0c5668116 swap 'pkg push' for 'pkg build --push', keeping 'pkg push' as deprecated but still working (#4141)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-07-04 18:00:28 +03:00
Avi Deitcher
2b4687338b add support for pkg build authentication (#4137)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-07-02 18:52:05 +03:00
Avi Deitcher
940c1b7b3b simplify cache locking (#4136)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-06-30 20:58:50 +03:00
Daniel S.
818bccf20f docs: Add instructions for OCI export from Docker (#4135)
Signed-off-by: Daniel Smith <daniel@razorsecure.com>
2025-06-30 16:27:54 +03:00
Avi Deitcher
50120bce2d ensure that new index does not break on missing lock file (#4134)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-06-27 11:01:43 +03:00
Avi Deitcher
254aefc953 check for dirty tree without update-index, which is not parallel-safe (#4133)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-06-26 19:53:13 +03:00
Avi Deitcher
4df360d62d Centralize safe cache writes (#4132)
* centralize all writing of the index.json to one place

Signed-off-by: Avi Deitcher <avi@deitcher.net>

* create filelock utility

Signed-off-by: Avi Deitcher <avi@deitcher.net>

* leverage file locks for cache index.json

Signed-off-by: Avi Deitcher <avi@deitcher.net>

---------

Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-06-26 19:02:49 +03:00
christoph-zededa
3f54a80824 git: synchronize update-index with a mutex (#4130)
if `pkglib.NewFromConfig` is used in parallel, it calls
```
git -C /some/directory update-index -q --refresh
```
in parallel.

But `git` does not like this and exits with 128.

This can be easily tried with:
```
git -C /some/dir update-index -q --refresh & \
git -C /some/dir update-index -q --refresh
```

Signed-off-by: Christoph Ostarek <christoph@zededa.com>
2025-06-25 21:59:47 +03:00
Avi Deitcher
d45d3e8c6e more builder race condition; do restart if could not remove, and only go via container ID (#4129)
Signed-off-by: Avi Deitcher <avi@deitcher.net>
2025-06-25 21:21:51 +03:00
305 changed files with 37719 additions and 342 deletions

View File

@@ -59,3 +59,31 @@ is provided, it always will pull, independent of what is in the cache.
The read process is smart enough to check each blob in the local cache before downloading
it from a registry.
## Imports from local Docker instance
To import an image from your local Docker daemon into LinuxKit, youll need to ensure the image is exported in the [OCI image format](https://docs.docker.com/build/exporters/oci-docker/), which LinuxKit understands.
This requires using a `docker-container` [buildx driver](https://docs.docker.com/build/builders/drivers/docker-container/), rather than the default.
Set it up like so:
```shell
docker buildx create --driver docker-container --driver-opt image=moby/buildkit:latest --name=ocibuilder --bootstrap
```
Then build and export your image using the OCI format:
```shell
docker buildx build --builder=ocibuilder --output type=oci,name=foo . > foo.tar
```
You can now import it into LinuxKit with:
```shell
linuxkit cache import foo.tar
```
Note that this process, as described, will only produce images for the platform/architecture you're currently on. To produce multi-platform images requires extra docker build flags and external builder or QEMU support - see [here](https://docs.docker.com/build/building/multi-platform/).
This workaround is only necessary when working with the local Docker daemon. If youre pulling from Docker Hub or another registry, you dont need to do any of this.

5
src/cmd/linuxkit/cache/const.go vendored Normal file
View File

@@ -0,0 +1,5 @@
package cache
const (
lockfile = ".lock"
)

View File

@@ -2,20 +2,44 @@ package cache
import (
"fmt"
"os"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
)
// Get get or initialize the cache
func Get(cache string) (layout.Path, error) {
func (p *Provider) Get(cache string) (layout.Path, error) {
// ensure the dir exists
if err := os.MkdirAll(cache, os.ModePerm); err != nil {
return "", fmt.Errorf("unable to create cache directory %s: %v", cache, err)
}
// first try to read the layout from the path
// if it exists, we can use it
// if it does not exist, we will initialize it
//
// do not lock for first read, because we do not need the lock except for initialization
// and future writes, so why slow down reads?
l, err := layout.FromPath(cache)
// initialize the cache path if needed
p, err := layout.FromPath(cache)
if err != nil {
p, err = layout.Write(cache, empty.Index)
if err := p.Lock(); err != nil {
return "", fmt.Errorf("unable to lock cache %s: %v", cache, err)
}
defer p.Unlock()
// after lock, try to read the layout again
// in case another process initialized it while we were waiting for the lock
// if it still does not exist, we will initialize it
l, err = layout.FromPath(cache)
if err != nil {
return p, fmt.Errorf("could not initialize cache at path %s: %v", cache, err)
l, err = layout.Write(cache, empty.Index)
if err != nil {
return l, fmt.Errorf("could not initialize cache at path %s: %v", cache, err)
}
}
}
return p, nil
return l, nil
}

View File

@@ -1,20 +1,30 @@
package cache
import (
"fmt"
"path/filepath"
"sync"
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/plugins/content/local"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
log "github.com/sirupsen/logrus"
)
// Provider cache implementation of cacheProvider
type Provider struct {
cache layout.Path
store content.Store
cache layout.Path
store content.Store
dir string
lock *util.FileLock
lockMut sync.Mutex
}
// NewProvider create a new CacheProvider based in the provided directory
func NewProvider(dir string) (*Provider, error) {
p, err := Get(dir)
p := &Provider{dir: dir, lockMut: sync.Mutex{}}
layout, err := p.Get(dir)
if err != nil {
return nil, err
}
@@ -22,5 +32,39 @@ func NewProvider(dir string) (*Provider, error) {
if err != nil {
return nil, err
}
return &Provider{p, store}, nil
p.cache = layout
p.store = store
return p, nil
}
// Lock locks the cache directory to prevent concurrent access
func (p *Provider) Lock() error {
// if the lock is already set, we do not need to do anything
if p.lock != nil {
return nil
}
p.lockMut.Lock()
defer p.lockMut.Unlock()
var lockFile = filepath.Join(p.dir, lockfile)
lock, err := util.Lock(lockFile)
if err != nil {
return fmt.Errorf("unable to retrieve cache lock %s: %v", lockFile, err)
}
p.lock = lock
return nil
}
// Unlock releases the lock on the cache directory
func (p *Provider) Unlock() {
p.lockMut.Lock()
defer p.lockMut.Unlock()
// if the lock is not set, we do not need to do anything
if p.lock == nil {
return
}
var lockFile = filepath.Join(p.dir, lockfile)
if err := p.lock.Unlock(); err != nil {
log.Errorf("unable to close lock for cache %s: %v", lockFile, err)
}
p.lock = nil
}

View File

@@ -9,8 +9,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
namepkg "github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/validate"
@@ -182,10 +180,11 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
return fmt.Errorf("error getting manifest for trusted image %s: %v", name, err)
}
// use the original image name in the annotation
annotations := map[string]string{
imagespec.AnnotationRefName: fullname,
// lock the cache so we can write to it
if err := p.Lock(); err != nil {
return fmt.Errorf("unable to lock cache for writing: %v", err)
}
defer p.Unlock()
// first attempt as an index
ii, err := desc.ImageIndex()
@@ -195,7 +194,7 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
if err := p.cache.WriteIndex(ii); err != nil {
return fmt.Errorf("unable to write index: %v", err)
}
if err := p.DescriptorWrite(&v1ref, desc.Descriptor); err != nil {
if err := p.DescriptorWrite(v1ref.String(), desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
if withArchReferences {
@@ -206,11 +205,10 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
for _, m := range im.Manifests {
if m.MediaType.IsImage() && m.Platform != nil && m.Platform.Architecture != unknown && m.Platform.OS != unknown {
archSpecific := fmt.Sprintf("%s-%s", ref.String(), m.Platform.Architecture)
archRef, err := reference.Parse(archSpecific)
if err != nil {
if _, err := reference.Parse(archSpecific); err != nil {
return fmt.Errorf("unable to parse arch-specific reference %s: %v", archSpecific, err)
}
if err := p.DescriptorWrite(&archRef, m); err != nil {
if err := p.DescriptorWrite(archSpecific, m); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
}
@@ -224,9 +222,12 @@ func (p *Provider) Pull(name string, withArchReferences bool) error {
return fmt.Errorf("provided image is neither an image nor an index: %s", name)
}
log.Debugf("ImageWrite retrieved %s is image, saving", fullname)
if err = p.cache.ReplaceImage(im, match.Name(fullname), layout.WithAnnotations(annotations)); err != nil {
if err = p.cache.WriteImage(im); err != nil {
return fmt.Errorf("unable to save image to cache: %v", err)
}
if err = p.DescriptorWrite(fullname, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write updated descriptor to cache: %v", err)
}
}
return nil

View File

@@ -12,7 +12,6 @@ import (
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
log "github.com/sirupsen/logrus"
)
@@ -67,7 +66,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
}
desc, err := remote.Get(ref, remoteOptions...)
if err == nil && desc != nil && dig == desc.Digest {
fmt.Printf("%s image already available on remote registry, skipping push", remoteName)
fmt.Printf("%s image already available on remote registry, skipping push\n", remoteName)
return nil
}
log.Debugf("pushing image %s as %s", name, remoteName)
@@ -89,7 +88,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
desc, err := remote.Get(ref, remoteOptions...)
if err == nil && desc != nil {
if dig == desc.Digest {
fmt.Printf("%s index already available on remote registry, skipping push", remoteName)
fmt.Printf("%s index already available on remote registry, skipping push\n", remoteName)
return nil
}
// we have a different index, need to cross-reference and only override relevant stuff
@@ -131,11 +130,7 @@ func (p *Provider) Push(name, remoteName string, withArchSpecificTags, override
// it might not have existed, so we can add it locally
// use the original image name in the annotation
desc := m.DeepCopy()
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = archTag
if err := p.cache.AppendDescriptor(*desc); err != nil {
if err := p.DescriptorWrite(archTag, *desc); err != nil {
return fmt.Errorf("error appending descriptor for %s to layout index: %v", archTag, err)
}
img, err = p.cache.Image(m.Digest)

View File

@@ -65,7 +65,7 @@ func (p *Provider) Remove(name string) error {
log.Warnf("unable to remove blob %s for %s: %v", blob, name, err)
}
}
return p.cache.RemoveDescriptors(match.Name(name))
return p.RemoveDescriptors(match.Name(name))
}
func blobsForImage(img v1.Image) ([]v1.Hash, error) {

View File

@@ -14,7 +14,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/match"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
@@ -77,10 +76,11 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
return fmt.Errorf("error getting manifest for image %s: %v", pullImageName, err)
}
// use the original image name in the annotation
annotations := map[string]string{
imagespec.AnnotationRefName: image,
// get our lock
if err := p.Lock(); err != nil {
return fmt.Errorf("unable to lock cache for removing descriptors: %v", err)
}
defer p.Unlock()
// first attempt as an index
ii, err := desc.ImageIndex()
@@ -120,7 +120,7 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
if err := p.cache.WriteIndex(ii); err != nil {
return fmt.Errorf("unable to write index: %v", err)
}
if err := p.DescriptorWrite(ref, desc.Descriptor); err != nil {
if err := p.DescriptorWrite(ref.String(), desc.Descriptor); err != nil {
return fmt.Errorf("unable to write index descriptor to cache: %v", err)
}
} else {
@@ -131,8 +131,11 @@ func (p *Provider) ImagePull(ref *reference.Spec, platforms []imagespec.Platform
return fmt.Errorf("provided image is neither an image nor an index: %s", image)
}
log.Debugf("ImageWrite retrieved %s is image, saving", pullImageName)
if err = p.cache.ReplaceImage(im, match.Name(image), layout.WithAnnotations(annotations)); err != nil {
return fmt.Errorf("unable to save image to cache: %v", err)
if err := p.cache.WriteImage(im); err != nil {
return fmt.Errorf("error writing image %s to cache: %v", pullImageName, err)
}
if err := p.DescriptorWrite(image, desc.Descriptor); err != nil {
return fmt.Errorf("unable to write image descriptor to cache: %v", err)
}
}
return nil
@@ -149,6 +152,11 @@ func (p *Provider) ImageLoad(r io.Reader) ([]v1.Descriptor, error) {
index bytes.Buffer
)
log.Debugf("ImageWriteTar to cache")
// get our lock
if err := p.Lock(); err != nil {
return nil, fmt.Errorf("unable to lock cache: %v", err)
}
defer p.Unlock()
for {
header, err := tr.Next()
if err == io.EOF {
@@ -208,20 +216,11 @@ func (p *Provider) ImageLoad(r io.Reader) ([]v1.Descriptor, error) {
// each of these is either an image or an index
// either way, it gets added directly to the linuxkit cache index.
for _, desc := range im.Manifests {
if imgName, ok := desc.Annotations[images.AnnotationImageName]; ok {
// remove the old descriptor, if it exists
if err := p.cache.RemoveDescriptors(match.Name(imgName)); err != nil {
return nil, fmt.Errorf("unable to remove old descriptors for %s: %v", imgName, err)
imgName, ok := desc.Annotations[images.AnnotationImageName]
if ok {
if err := p.DescriptorWrite(imgName, desc); err != nil {
return nil, fmt.Errorf("error writing descriptor for %s: %v", imgName, err)
}
// save the image name under our proper annotation
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = imgName
}
log.Debugf("appending descriptor %#v", desc)
if err := p.cache.AppendDescriptor(desc); err != nil {
return nil, fmt.Errorf("error appending descriptor to layout index: %v", err)
}
descs = append(descs, desc)
}
@@ -256,6 +255,11 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
return fmt.Errorf("error parsing index: %v", err)
}
var im v1.IndexManifest
// get our lock
if err := p.Lock(); err != nil {
return fmt.Errorf("unable to lock cache: %v", err)
}
defer p.Unlock()
// do we update an existing one? Or create a new one?
if len(indexes) > 0 {
// we already had one, so update just the referenced index and return
@@ -366,9 +370,6 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
return fmt.Errorf("error writing new index to json: %v", err)
}
// finally update the descriptor in the root
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptor from index.json: %v", err)
}
desc := v1.Descriptor{
MediaType: types.OCIImageIndex,
Size: size,
@@ -377,36 +378,7 @@ func (p *Provider) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor)
imagespec.AnnotationRefName: image,
},
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor to index.json: %v", err)
}
return nil
}
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) error {
if ref == nil {
return errors.New("cannot write descriptor without reference name")
}
image := ref.String()
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = image
log.Debugf("writing descriptor for image %s", image)
// do we update an existing one? Or create a new one?
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
}
return nil
return p.DescriptorWrite(ref.String(), desc)
}
func (p *Provider) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
@@ -496,3 +468,46 @@ func (p *Provider) ImageInRegistry(ref *reference.Spec, trustedRef, architecture
}
return false, nil
}
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
func (p *Provider) DescriptorWrite(image string, desc v1.Descriptor) error {
if image == "" {
return errors.New("cannot write descriptor without reference name")
}
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[imagespec.AnnotationRefName] = image
log.Debugf("writing descriptor for image %s", image)
// get our lock
if err := p.Lock(); err != nil {
return fmt.Errorf("unable to lock cache for writing descriptors: %v", err)
}
defer p.Unlock()
// get our lock
// do we update an existing one? Or create a new one?
if err := p.cache.RemoveDescriptors(match.Name(image)); err != nil {
return fmt.Errorf("unable to remove old descriptors for %s: %v", image, err)
}
if err := p.cache.AppendDescriptor(desc); err != nil {
return fmt.Errorf("unable to append new descriptor for %s: %v", image, err)
}
return nil
}
// RemoveDescriptors removes all descriptors that match the provided matcher.
// It does so in a parallel-access-safe way
func (p *Provider) RemoveDescriptors(matcher match.Matcher) error {
// get our lock
if err := p.Lock(); err != nil {
return fmt.Errorf("unable to lock cache for removing descriptors: %v", err)
}
defer p.Unlock()
return p.cache.RemoveDescriptors(matcher)
}

View File

@@ -72,6 +72,7 @@ require (
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/containerd/console v1.0.5 // indirect
github.com/containerd/containerd/api v1.9.0 // indirect
github.com/containerd/continuity v0.4.5 // indirect
@@ -89,6 +90,7 @@ require (
github.com/docker/docker-credential-helpers v0.9.3 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fvbommel/sortorder v1.0.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofrs/flock v0.12.1 // indirect
@@ -110,22 +112,28 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/signal v0.7.1 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect
github.com/shibumi/go-pathspec v1.3.0 // indirect
github.com/smartystreets/goconvey v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 // indirect
github.com/tonistiigi/fsutil v0.0.0-20250605211040-586307ad452f // indirect
github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
@@ -139,9 +147,12 @@ require (
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/sdk v1.35.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
golang.org/x/mod v0.24.0 // indirect

View File

@@ -41,9 +41,13 @@ github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1
github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
github.com/aws/aws-sdk-go v1.44.82 h1:Miji7nHIMxTWfa831nZf8XAcMWGLaT+PvsS6CdbMG7M=
github.com/aws/aws-sdk-go v1.44.82/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
@@ -102,8 +106,12 @@ github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5Jxw
github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8=
github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo=
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -114,6 +122,8 @@ github.com/equinix/equinix-sdk-go v0.42.0 h1:jjgdFs0rx6nOwsu/dLh6ImopD0M1Rn7QIn9
github.com/equinix/equinix-sdk-go v0.42.0/go.mod h1:hEb3XLaedz7xhl/dpPIS6eOIiXNPeqNiVoyDrT6paIg=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE=
github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -172,6 +182,8 @@ github.com/gophercloud/utils v0.0.0-20181029231510-34f5991525d1 h1:/UC0TO8h4XaxQ
github.com/gophercloud/utils v0.0.0-20181029231510-34f5991525d1/go.mod h1:wjDF8z83zTeg5eMLml5EBSlAhbF7G8DobyI1YsMuyzw=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA=
@@ -207,6 +219,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 h1:jUp75lepDg0phMUJBCmvaeFDldD2N3S1lBuPwUTszio=
github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8=
@@ -245,6 +261,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/moul/gotty-client v1.7.1-0.20180526075433-e5589f6df359 h1:gIpgn2fMLvlbrBEXoQje5zrywZgst7CyXMY/S/L+OH0=
github.com/moul/gotty-client v1.7.1-0.20180526075433-e5589f6df359/go.mod h1:CxM/JGtpRrEPve5H04IhxJrGhxgwxMc6jSP2T4YD60w=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -263,13 +281,21 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/radu-matei/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible h1:lh3lhjZXXLv7/8cJKQQtgWLJnszuN1xYD9QHTf/dKgE=
github.com/radu-matei/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible/go.mod h1:dK8sDtj2CPkz+pRG4nNF7+WzmyYjtJwfYrZApxB+zFg=
github.com/radu-matei/azure-vhd-utils v0.0.0-20170531165126-e52754d5569d h1:9aQ38JpJEOfipvLKM/NZ2MWxpZIlZgR+6AjotW+a2T8=
github.com/radu-matei/azure-vhd-utils v0.0.0-20170531165126-e52754d5569d/go.mod h1:gdClACBl2wyQrk+AgrDhjviWZiI9oQz7YrHQ9t/hq/Y=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rn/iso9660wrap v0.0.0-20171120145750-baf8d62ad315 h1:DjbO/+j3556fy07xoEM/MyLYN3WwwYyt4dHRC5U+KN8=
github.com/rn/iso9660wrap v0.0.0-20171120145750-baf8d62ad315/go.mod h1:qrZfINtl+sTGgS3elQWqWsD2Ke4Il5jDzBr2Q+lzuuE=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
@@ -309,6 +335,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/surma/gocpio v1.0.2-0.20160926205914-fcb68777e7dc h1:iA3Eg1OVd2o0M4M+0PBsBBssMz98L8CUH7x0xVkuyUA=
github.com/surma/gocpio v1.0.2-0.20160926205914-fcb68777e7dc/go.mod h1:zaLNaN+EDnfSnNdWPJJf9OZxWF817w5dt8JNzF9LCVI=
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 h1:r0p7fK56l8WPequOaR3i9LBqfPtEdXIQbUTzT55iqT4=
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY=
github.com/tonistiigi/fsutil v0.0.0-20250605211040-586307ad452f h1:MoxeMfHAe5Qj/ySSBfL8A7l1V+hxuluj8owsIEEZipI=
github.com/tonistiigi/fsutil v0.0.0-20250605211040-586307ad452f/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98=
github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 h1:2f304B10LaZdB8kkVEaoXvAMVan2tl9AiK4G0odjQtE=
@@ -342,8 +372,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRND
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 h1:m639+BofXTvcY1q8CGs4ItwQarYtJPOWmVobfM1HpVI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0/go.mod h1:LjReUci/F4BUyv+y4dwnq3h/26iNOeC3wAIqgvTIZVo=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk=
go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=

View File

@@ -7,7 +7,10 @@ import (
"github.com/spf13/cobra"
)
var pkglibConfig pkglib.PkglibConfig
var (
pkglibConfig pkglib.PkglibConfig
registryCreds []string
)
func pkgCmd() *cobra.Command {
var (
@@ -72,9 +75,11 @@ func pkgCmd() *cobra.Command {
},
}
cmd.AddCommand(pkgBuildCmd())
// because there is an alias 'pkg push' for 'pkg build --push', we need to add the build command first
buildCmd := pkgBuildCmd()
cmd.AddCommand(buildCmd)
cmd.AddCommand(pkgBuilderCmd())
cmd.AddCommand(pkgPushCmd())
cmd.AddCommand(pkgPushCmd(buildCmd))
cmd.AddCommand(pkgShowTagCmd())
cmd.AddCommand(pkgManifestCmd())
cmd.AddCommand(pkgRemoteTagCmd())
@@ -96,5 +101,6 @@ func pkgCmd() *cobra.Command {
cmd.PersistentFlags().BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty")
cmd.PersistentFlags().BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively")
cmd.PersistentFlags().StringSliceVar(&registryCreds, "registry-creds", nil, "Registry auths to use for building images, format is <registry>=<username>:<password> OR <registry>=<registry-token-base64>; do NOT forget to base64 encode it. If no username is provided, it is treated as a registry token. <registry> must be a URL, e.g. 'https://index.docker.io/'. May be provided as many times as desired. Will override anything in your default.")
return cmd
}

View File

@@ -8,6 +8,7 @@ import (
"strings"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
)
@@ -19,27 +20,28 @@ const (
)
// some logic clarification:
// pkg build - builds unless is in cache or published in registry
// pkg build --pull - builds unless is in cache or published in registry; pulls from registry if not in cache
// pkg build --force - always builds even if is in cache or published in registry
// pkg build --force --pull - always builds even if is in cache or published in registry; --pull ignored
// pkg push - always builds unless is in cache
// pkg push --force - always builds even if is in cache
// pkg push --nobuild - skips build; if not in cache, fails
// pkg push --nobuild --force - nonsensical
// pkg build - builds unless is in cache or published in registry
// pkg build --pull - builds unless is in cache or published in registry; pulls from registry if not in cache
// pkg build --force - always builds even if is in cache or published in registry
// pkg build --force --pull - always builds even if is in cache or published in registry; --pull ignored
// pkg build --push - always builds unless is in cache or published in registry; pushes to registry
// pkg build --push --force - always builds even if is in cache
// pkg build --push --nobuild - skips build; if not in cache, fails
// pkg build --push --nobuild --force - nonsensical
// pkg push - equivalent to pkg build --push
// addCmdRunPkgBuildPush adds the RunE function and flags to a cobra.Command
// for "pkg build" or "pkg push".
func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
func pkgBuildCmd() *cobra.Command {
var (
force bool
pull bool
push bool
ignoreCache bool
docker bool
platforms string
skipPlatforms string
builders string
builderImage string
builderConfig string
builderRestart bool
release string
nobuild bool
@@ -51,179 +53,235 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
progress string
ssh []string
)
cmd.RunE = func(cmd *cobra.Command, args []string) error {
pkgs, err := pkglib.NewFromConfig(pkglibConfig, args...)
if err != nil {
return err
}
if nobuild && force {
return errors.New("flags -force and -nobuild conflict")
}
if pull && force {
return errors.New("flags -force and -pull conflict")
}
var opts []pkglib.BuildOpt
if force {
opts = append(opts, pkglib.WithBuildForce())
}
if ignoreCache {
opts = append(opts, pkglib.WithBuildIgnoreCache())
}
if pull {
opts = append(opts, pkglib.WithBuildPull())
}
opts = append(opts, pkglib.WithBuildCacheDir(cacheDir.String()))
if withPush {
opts = append(opts, pkglib.WithBuildPush())
if nobuild {
opts = append(opts, pkglib.WithBuildSkip())
}
if release != "" {
opts = append(opts, pkglib.WithRelease(release))
}
if manifest {
opts = append(opts, pkglib.WithBuildManifest())
}
}
if docker {
opts = append(opts, pkglib.WithBuildTargetDockerCache())
}
if sbomScanner != "false" {
opts = append(opts, pkglib.WithBuildSbomScanner(sbomScanner))
}
opts = append(opts, pkglib.WithDockerfile(dockerfile))
// read any build arg files
var buildArgs []string
for _, filename := range buildArgFiles {
f, err := os.Open(filename)
cmd := &cobra.Command{
Use: "build",
Short: "build an OCI package from a directory with a yaml configuration file",
Long: `Build an OCI package from a directory with a yaml configuration file.
'path' specifies the path to the package source directory.
`,
Example: ` linuxkit pkg build [options] pkg/dir/`,
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
pkgs, err := pkglib.NewFromConfig(pkglibConfig, args...)
if err != nil {
return fmt.Errorf("error opening build args file %s: %w", filename, err)
return err
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
buildArgs = append(buildArgs, scanner.Text())
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("error reading build args file %s: %w", filename, err)
}
}
opts = append(opts, pkglib.WithBuildArgs(buildArgs))
// skipPlatformsMap contains platforms that should be skipped
skipPlatformsMap := make(map[string]bool)
if skipPlatforms != "" {
for _, platform := range strings.Split(skipPlatforms, ",") {
parts := strings.SplitN(platform, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[0] != "linux" || parts[1] == "" {
return fmt.Errorf("invalid target platform specification '%s'", platform)
if nobuild && force {
return errors.New("flags -force and -nobuild conflict")
}
if pull && force {
return errors.New("flags -force and -pull conflict")
}
var opts []pkglib.BuildOpt
if force {
opts = append(opts, pkglib.WithBuildForce())
}
if ignoreCache {
opts = append(opts, pkglib.WithBuildIgnoreCache())
}
if pull {
opts = append(opts, pkglib.WithBuildPull())
}
opts = append(opts, pkglib.WithBuildCacheDir(cacheDir.String()))
if push {
opts = append(opts, pkglib.WithBuildPush())
if nobuild {
opts = append(opts, pkglib.WithBuildSkip())
}
skipPlatformsMap[strings.Trim(parts[1], " ")] = true
}
}
// if requested specific platforms, build those. If not, then we will
// retrieve the defaults in the loop over each package.
var plats []imagespec.Platform
// don't allow the use of --skip-platforms with --platforms
if platforms != "" && skipPlatforms != "" {
return errors.New("--skip-platforms and --platforms may not be used together")
}
// process the platforms if provided
if platforms != "" {
for _, p := range strings.Split(platforms, ",") {
parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", p)
os.Exit(1)
if release != "" {
opts = append(opts, pkglib.WithRelease(release))
}
if manifest {
opts = append(opts, pkglib.WithBuildManifest())
}
plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
}
}
if docker {
opts = append(opts, pkglib.WithBuildTargetDockerCache())
}
// build the builders map
buildersMap := map[string]string{}
// look for builders env var
buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap)
if err != nil {
return fmt.Errorf("error in environment variable %s: %w", buildersEnvVar, err)
}
// any CLI options override env var
buildersMap, err = buildPlatformBuildersMap(builders, buildersMap)
if err != nil {
return fmt.Errorf("error in --builders flag: %w", err)
}
opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
opts = append(opts, pkglib.WithBuildBuilderImage(builderImage))
opts = append(opts, pkglib.WithBuildBuilderRestart(builderRestart))
opts = append(opts, pkglib.WithProgress(progress))
if len(ssh) > 0 {
opts = append(opts, pkglib.WithSSH(ssh))
}
if sbomScanner != "false" {
opts = append(opts, pkglib.WithBuildSbomScanner(sbomScanner))
}
opts = append(opts, pkglib.WithDockerfile(dockerfile))
for _, p := range pkgs {
// things we need our own copies of
var (
pkgOpts = make([]pkglib.BuildOpt, len(opts))
pkgPlats = make([]imagespec.Platform, len(plats))
)
copy(pkgOpts, opts)
copy(pkgPlats, plats)
// unless overridden, platforms are specific to a package, so this needs to be inside the for loop
if len(pkgPlats) == 0 {
for _, a := range p.Arches() {
if _, ok := skipPlatformsMap[a]; ok {
continue
// read any build arg files
var buildArgs []string
for _, filename := range buildArgFiles {
f, err := os.Open(filename)
if err != nil {
return fmt.Errorf("error opening build args file %s: %w", filename, err)
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
buildArgs = append(buildArgs, scanner.Text())
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("error reading build args file %s: %w", filename, err)
}
}
opts = append(opts, pkglib.WithBuildArgs(buildArgs))
// skipPlatformsMap contains platforms that should be skipped
skipPlatformsMap := make(map[string]bool)
if skipPlatforms != "" {
for _, platform := range strings.Split(skipPlatforms, ",") {
parts := strings.SplitN(platform, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[0] != "linux" || parts[1] == "" {
return fmt.Errorf("invalid target platform specification '%s'", platform)
}
pkgPlats = append(pkgPlats, imagespec.Platform{OS: "linux", Architecture: a})
skipPlatformsMap[strings.Trim(parts[1], " ")] = true
}
}
// if requested specific platforms, build those. If not, then we will
// retrieve the defaults in the loop over each package.
var plats []imagespec.Platform
// don't allow the use of --skip-platforms with --platforms
if platforms != "" && skipPlatforms != "" {
return errors.New("--skip-platforms and --platforms may not be used together")
}
// process the platforms if provided
if platforms != "" {
for _, p := range strings.Split(platforms, ",") {
parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", p)
os.Exit(1)
}
plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
}
}
// if there are no platforms to build for, do nothing.
// note that this is *not* an error; we simply skip it
if len(pkgPlats) == 0 {
fmt.Printf("Skipping %s with no architectures to build\n", p.Tag())
continue
// build the builders map
buildersMap := map[string]string{}
// look for builders env var
buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap)
if err != nil {
return fmt.Errorf("error in environment variable %s: %w", buildersEnvVar, err)
}
// any CLI options override env var
buildersMap, err = buildPlatformBuildersMap(builders, buildersMap)
if err != nil {
return fmt.Errorf("error in --builders flag: %w", err)
}
if builderConfig != "" {
if _, err := os.Stat(builderConfig); err != nil {
return fmt.Errorf("error reading builder config file %s: %w", builderConfig, err)
}
opts = append(opts, pkglib.WithBuildBuilderConfig(builderConfig))
}
pkgOpts = append(pkgOpts, pkglib.WithBuildPlatforms(pkgPlats...))
var msg, action string
switch {
case !withPush:
msg = fmt.Sprintf("Building %q", p.Tag())
action = "building"
case nobuild:
msg = fmt.Sprintf("Pushing %q without building", p.Tag())
action = "building and pushing"
default:
msg = fmt.Sprintf("Building and pushing %q", p.Tag())
action = "building and pushing"
opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
opts = append(opts, pkglib.WithBuildBuilderImage(builderImage))
opts = append(opts, pkglib.WithBuildBuilderRestart(builderRestart))
opts = append(opts, pkglib.WithProgress(progress))
if len(ssh) > 0 {
opts = append(opts, pkglib.WithSSH(ssh))
}
if len(registryCreds) > 0 {
registryCredMap := make(map[string]spec.RegistryAuth)
for _, cred := range registryCreds {
parts := strings.SplitN(cred, "=", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return fmt.Errorf("invalid registry auth specification '%s'", cred)
}
registryPart := strings.TrimSpace(parts[0])
authPart := strings.TrimSpace(parts[1])
var auth spec.RegistryAuth
// if the auth is a token, we don't need a username
credParts := strings.SplitN(authPart, ":", 2)
var userPart, credPart string
userPart = strings.TrimSpace(credParts[0])
if len(credParts) == 2 {
credPart = strings.TrimSpace(credParts[1])
}
switch {
case len(registryPart) == 0:
return fmt.Errorf("invalid registry auth specification '%s', registry must not be blank", cred)
case len(credParts) == 2 && (len(userPart) == 0 || len(credPart) == 0):
return fmt.Errorf("invalid registry auth specification '%s', username and password must not be blank", cred)
case len(credParts) == 1 && len(userPart) == 0:
return fmt.Errorf("invalid registry auth specification '%s', token must not be blank", cred)
case len(credParts) == 2:
auth = spec.RegistryAuth{
Username: userPart,
Password: credPart,
}
case len(credParts) == 1:
auth = spec.RegistryAuth{
RegistryToken: authPart,
}
default:
return fmt.Errorf("invalid registry auth specification '%s'", cred)
}
registryCredMap[registryPart] = auth
}
opts = append(opts, pkglib.WithRegistryAuth(registryCredMap))
}
fmt.Println(msg)
for _, p := range pkgs {
// things we need our own copies of
var (
pkgOpts = make([]pkglib.BuildOpt, len(opts))
pkgPlats = make([]imagespec.Platform, len(plats))
)
copy(pkgOpts, opts)
copy(pkgPlats, plats)
// unless overridden, platforms are specific to a package, so this needs to be inside the for loop
if len(pkgPlats) == 0 {
for _, a := range p.Arches() {
if _, ok := skipPlatformsMap[a]; ok {
continue
}
pkgPlats = append(pkgPlats, imagespec.Platform{OS: "linux", Architecture: a})
}
}
if err := p.Build(pkgOpts...); err != nil {
return fmt.Errorf("error %s %q: %w", action, p.Tag(), err)
// if there are no platforms to build for, do nothing.
// note that this is *not* an error; we simply skip it
if len(pkgPlats) == 0 {
fmt.Printf("Skipping %s with no architectures to build\n", p.Tag())
continue
}
pkgOpts = append(pkgOpts, pkglib.WithBuildPlatforms(pkgPlats...))
var msg, action string
switch {
case !push:
msg = fmt.Sprintf("Building %q", p.Tag())
action = "building"
case nobuild:
msg = fmt.Sprintf("Pushing %q without building", p.Tag())
action = "building and pushing"
default:
msg = fmt.Sprintf("Building and pushing %q", p.Tag())
action = "building and pushing"
}
fmt.Println(msg)
if err := p.Build(pkgOpts...); err != nil {
return fmt.Errorf("error %s %q: %w", action, p.Tag(), err)
}
}
}
return nil
return nil
},
}
cmd.Flags().BoolVar(&force, "force", false, "Force rebuild even if image is in local cache")
cmd.Flags().BoolVar(&pull, "pull", false, "Pull image if in registry but not in local cache; conflicts with --force")
cmd.Flags().BoolVar(&push, "push", false, "After building, if successful, push the image to the registry; if --nobuild is set, just push")
cmd.Flags().BoolVar(&ignoreCache, "ignore-cached", false, "Ignore cached intermediate images, always pulling from registry")
cmd.Flags().BoolVar(&docker, "docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
cmd.Flags().StringVar(&platforms, "platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
cmd.Flags().StringVar(&skipPlatforms, "skip-platforms", "", "Platforms that should be skipped, even if present in build.yml")
cmd.Flags().StringVar(&builders, "builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
cmd.Flags().StringVar(&builderImage, "builder-image", defaultBuilderImage, "buildkit builder container image to use")
cmd.Flags().StringVar(&builderConfig, "builder-config", "", "path to buildkit builder config.toml file to use, overrides the default config.toml in the builder image. When provided, copied over into builder, along with all certs. Use paths for certificates relative to your local host, they will be adjusted on copying into the container. USE WITH CAUTION")
cmd.Flags().BoolVar(&builderRestart, "builder-restart", false, "force restarting builder, even if container with correct name and image exists")
cmd.Flags().Var(&cacheDir, "cache", fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))
cmd.Flags().StringVar(&release, "release", "", "Release the given version")
@@ -237,18 +295,6 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
return cmd
}
func pkgBuildCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "build",
Short: "build an OCI package from a directory with a yaml configuration file",
Long: `Build an OCI package from a directory with a yaml configuration file.
'path' specifies the path to the package source directory.
`,
Example: ` linuxkit pkg build [options] pkg/dir/`,
Args: cobra.MinimumNArgs(1),
}
return addCmdRunPkgBuildPush(cmd, false)
}
func buildPlatformBuildersMap(inputs string, existing map[string]string) (map[string]string, error) {
if inputs == "" {

View File

@@ -12,9 +12,10 @@ import (
func pkgBuilderCmd() *cobra.Command {
var (
builders string
platforms string
builderImage string
builders string
platforms string
builderImage string
builderConfigPath string
)
cmd := &cobra.Command{
Use: "builder",
@@ -40,11 +41,11 @@ func pkgBuilderCmd() *cobra.Command {
platformsToClean := strings.Split(platforms, ",")
switch command {
case "du":
if err := pkglib.DiskUsage(buildersMap, builderImage, platformsToClean, verbose); err != nil {
if err := pkglib.DiskUsage(buildersMap, builderImage, builderConfigPath, platformsToClean, verbose); err != nil {
return fmt.Errorf("unable to print disk usage of builder: %w", err)
}
case "prune":
if err := pkglib.PruneBuilder(buildersMap, builderImage, platformsToClean, verbose); err != nil {
if err := pkglib.PruneBuilder(buildersMap, builderImage, builderConfigPath, platformsToClean, verbose); err != nil {
return fmt.Errorf("unable to prune builder: %w", err)
}
default:
@@ -57,6 +58,7 @@ func pkgBuilderCmd() *cobra.Command {
cmd.PersistentFlags().StringVar(&builders, "builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
cmd.PersistentFlags().StringVar(&platforms, "platforms", fmt.Sprintf("linux/%s", runtime.GOARCH), "Which platforms we built images for")
cmd.PersistentFlags().StringVar(&builderImage, "builder-image", defaultBuilderImage, "buildkit builder container image to use")
cmd.Flags().StringVar(&builderConfigPath, "builder-config", "", "path to buildkit builder config.toml file to use, overrides the default config.toml in the builder image; USE WITH CAUTION")
return cmd
}

View File

@@ -1,18 +1,35 @@
package main
import "github.com/spf13/cobra"
import (
"fmt"
func pkgPushCmd() *cobra.Command {
"github.com/spf13/cobra"
)
func pkgPushCmd(buildCmd *cobra.Command) *cobra.Command {
cmd := &cobra.Command{
Use: "push",
Short: "build and push an OCI package from a directory with a yaml configuration file",
Short: "Alias for 'pkg build --push'",
Long: `Build and push an OCI package from a directory with a yaml configuration file.
'path' specifies the path to the package source directory.
The package may or may not be built first, depending on options
`,
Example: ` linuxkit pkg push [options] pkg/dir/`,
Args: cobra.MinimumNArgs(1),
Example: ` linuxkit pkg push [options] pkg/dir/`,
SuggestFor: []string{"build"},
Args: cobra.MinimumNArgs(1),
Deprecated: "use 'pkg build --push' instead",
RunE: func(cmd *cobra.Command, args []string) error {
// Create a copy of buildCmd with push=true
if err := buildCmd.Flags().Set("push", "true"); err != nil {
return fmt.Errorf("'pkg push' unable to set 'pkg build --push': %w", err)
}
// Pass the args to the build command
buildCmd.SetArgs(args)
return buildCmd.RunE(buildCmd, args)
},
}
return addCmdRunPkgBuildPush(cmd, true)
cmd.Flags().AddFlagSet(buildCmd.Flags())
return cmd
}

View File

@@ -24,28 +24,30 @@ import (
)
type buildOpts struct {
skipBuild bool
force bool
pull bool
ignoreCache bool
push bool
release string
manifest bool
targetDocker bool
cacheDir string
cacheProvider spec.CacheProvider
platforms []imagespec.Platform
builders map[string]string
runner dockerRunner
writer io.Writer
builderImage string
builderRestart bool
sbomScan bool
sbomScannerImage string
dockerfile string
buildArgs []string
progress string
ssh []string
skipBuild bool
force bool
pull bool
ignoreCache bool
push bool
release string
manifest bool
targetDocker bool
cacheDir string
cacheProvider spec.CacheProvider
platforms []imagespec.Platform
builders map[string]string
runner dockerRunner
writer io.Writer
builderImage string
builderConfigPath string
builderRestart bool
sbomScan bool
sbomScannerImage string
dockerfile string
buildArgs []string
progress string
ssh []string
registryAuth map[string]spec.RegistryAuth
}
// BuildOpt allows callers to specify options to Build
@@ -164,6 +166,14 @@ func WithBuildBuilderImage(image string) BuildOpt {
}
}
// WithBuildBuilderConfig set the contents of the
func WithBuildBuilderConfig(builderConfigPath string) BuildOpt {
return func(bo *buildOpts) error {
bo.builderConfigPath = builderConfigPath
return nil
}
}
// WithBuildBuilderRestart restart the builder container even if it already is running with the correct image version
func WithBuildBuilderRestart(restart bool) BuildOpt {
return func(bo *buildOpts) error {
@@ -223,6 +233,14 @@ func WithSSH(ssh []string) BuildOpt {
}
}
// WithRegistryAuth stores registry credentials
func WithRegistryAuth(creds map[string]spec.RegistryAuth) BuildOpt {
return func(bo *buildOpts) error {
bo.registryAuth = creds
return nil
}
}
// Build builds the package
func (p Pkg) Build(bos ...BuildOpt) error {
var bo buildOpts
@@ -449,10 +467,11 @@ func (p Pkg) Build(bos ...BuildOpt) error {
}
imageBuildOpts.SSH = bo.ssh
imageBuildOpts.RegistryAuths = bo.registryAuth
// build for each arch and save in the linuxkit cache
for _, platform := range platformsToBuild {
builtDescs, err := p.buildArch(ctx, d, c, bo.builderImage, platform.Architecture, bo.builderRestart, writer, bo, imageBuildOpts)
builtDescs, err := p.buildArch(ctx, d, c, bo.builderImage, bo.builderConfigPath, platform.Architecture, bo.builderRestart, writer, bo, imageBuildOpts)
if err != nil {
return fmt.Errorf("error building for arch %s: %v", platform.Architecture, err)
}
@@ -561,7 +580,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
if err != nil {
return err
}
if err := c.DescriptorWrite(&ref, *desc); err != nil {
if err := c.DescriptorWrite(fullRelTag, *desc); err != nil {
return err
}
if err := c.Push(fullRelTag, "", bo.manifest, true); err != nil {
@@ -602,7 +621,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
// C - manifest, saved in cache as is, referenced by the index (E), and returned as a descriptor
// D - attestations (if any), saved in cache as is, referenced by the index (E), and returned as a descriptor
// E - index, saved in cache as is, stored in cache as tag "image:tag-arch", *not* returned as a descriptor
func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c spec.CacheProvider, builderImage, arch string, restart bool, writer io.Writer, bo buildOpts, imageBuildOpts spec.ImageBuildOptions) ([]registry.Descriptor, error) {
func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c spec.CacheProvider, builderImage, builderConfigPath, arch string, restart bool, writer io.Writer, bo buildOpts, imageBuildOpts spec.ImageBuildOptions) ([]registry.Descriptor, error) {
var (
tagArch string
tag = p.FullTag()
@@ -671,7 +690,7 @@ func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c spec.CacheProvider
imageBuildOpts.Dockerfile = bo.dockerfile
if err := d.build(ctx, tagArch, p.path, builderName, builderImage, platform, restart, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
if err := d.build(ctx, tagArch, p.path, builderName, builderImage, builderConfigPath, platform, restart, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
stdoutCloser()
if strings.Contains(err.Error(), "executor failed running [/dev/.buildkit_qemu_emulator") {
return nil, fmt.Errorf("buildkit was unable to emulate %s. check binfmt has been set up and works for this platform: %v", platform, err)

View File

@@ -53,10 +53,10 @@ func (d *dockerMocker) contextSupportCheck() error {
}
return errors.New("contexts not supported")
}
func (d *dockerMocker) builder(_ context.Context, _, _, _ string, _ bool) (*buildkitClient.Client, error) {
func (d *dockerMocker) builder(_ context.Context, _, _, _, _ string, _ bool) (*buildkitClient.Client, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, builderRestart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, builderRestart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
if !d.enableBuild {
return errors.New("build disabled")
}
@@ -407,13 +407,12 @@ func (c *cacheMocker) Push(name, remoteName string, withManifest, override bool)
return nil
}
func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) error {
func (c *cacheMocker) DescriptorWrite(image string, desc v1.Descriptor) error {
if !c.enabledDescriptorWrite {
return errors.New("descriptor disabled")
}
var (
image = ref.String()
im = v1.IndexManifest{
im = v1.IndexManifest{
MediaType: types.OCIImageIndex,
Manifests: []v1.Descriptor{desc},
SchemaVersion: 2,

View File

@@ -5,9 +5,11 @@ package pkglib
//go:generate ./gen
import (
"archive/tar"
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -21,6 +23,7 @@ import (
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/pkg/reference"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/progress"
dockercontainertypes "github.com/docker/docker/api/types/container"
"github.com/google/go-containerregistry/pkg/authn"
@@ -37,12 +40,16 @@ import (
// golint requires comments on non-main(test)
// package for blank import
dockerconfig "github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/configfile"
dockerconfigtypes "github.com/docker/cli/cli/config/types"
_ "github.com/moby/buildkit/client/connhelper/dockercontainer"
_ "github.com/moby/buildkit/client/connhelper/ssh"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/linter"
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/session/upload/uploadprovider"
log "github.com/sirupsen/logrus"
@@ -54,16 +61,18 @@ const (
buildkitWaitServer = 30 // seconds
buildkitCheckInterval = 1 // seconds
sbomFrontEndKey = "attest:sbom"
buildkitConfigDir = "/etc/buildkit"
buildkitConfigPath = buildkitConfigDir + "/buildkitd.toml"
)
type dockerRunner interface {
tag(ref, tag string) error
build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
save(tgt string, refs ...string) error
load(src io.Reader) error
pull(img string) (bool, error)
contextSupportCheck() error
builder(ctx context.Context, dockerContext, builderImage, platform string, restart bool) (*buildkitClient.Client, error)
builder(ctx context.Context, dockerContext, builderImage, builderConfigPath, platform string, restart bool) (*buildkitClient.Client, error)
}
type dockerRunnerImpl struct {
@@ -218,14 +227,14 @@ func (dr *dockerRunnerImpl) contextSupportCheck() error {
// 1. if dockerContext is provided, try to create a builder with that context; if it succeeds, we are done; if not, return an error.
// 2. try to find an existing named runner with the pattern; if it succeeds, we are done; if not, try next.
// 3. try to create a generic builder using the default context named "linuxkit".
func (dr *dockerRunnerImpl) builder(ctx context.Context, dockerContext, builderImage, platform string, restart bool) (*buildkitClient.Client, error) {
func (dr *dockerRunnerImpl) builder(ctx context.Context, dockerContext, builderImage, builderConfigPath, platform string, restart bool) (*buildkitClient.Client, error) {
// if we were given a context, we must find a builder and use it, or create one and use it
if dockerContext != "" {
// does the context exist?
if err := dr.command(nil, io.Discard, io.Discard, "context", "inspect", dockerContext); err != nil {
return nil, fmt.Errorf("provided docker context '%s' not found", dockerContext)
}
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, platform, dockerContext, restart)
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, platform, dockerContext, restart)
if err != nil {
return nil, fmt.Errorf("error preparing builder based on context '%s': %v", dockerContext, err)
}
@@ -236,13 +245,13 @@ func (dr *dockerRunnerImpl) builder(ctx context.Context, dockerContext, builderI
dockerContext = fmt.Sprintf("%s-%s", "linuxkit", strings.ReplaceAll(platform, "/", "-"))
if err := dr.command(nil, io.Discard, io.Discard, "context", "inspect", dockerContext); err == nil {
// we found an appropriately named context, so let us try to use it or error out
if client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, platform, dockerContext, restart); err == nil {
if client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, platform, dockerContext, restart); err == nil {
return client, nil
}
}
// create a generic builder
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, "", "default", restart)
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, "", "default", restart)
if err != nil {
return nil, fmt.Errorf("error ensuring builder container in default context: %v", err)
}
@@ -254,7 +263,7 @@ func (dr *dockerRunnerImpl) builder(ctx context.Context, dockerContext, builderI
// but has the wrong version of buildkit, or not running buildkit at all, remove it and create an appropriate
// one.
// Returns a network connection to the buildkit builder in the container.
func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, image, platform, dockerContext string, forceRestart bool) (*buildkitClient.Client, error) {
func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, image, configPath, platform, dockerContext string, forceRestart bool) (*buildkitClient.Client, error) {
// if no error, then we have a builder already
// inspect it to make sure it is of the right type
var (
@@ -276,6 +285,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
)
for range buildKitCheckRetryCount {
var b bytes.Buffer
var cid string
if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
// we already have a container named "linuxkit-builder" in the provided context.
// get its state and config
@@ -284,8 +294,38 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
}
cid = containerJSON[0].ID
existingImage := containerJSON[0].Config.Image
isRunning := containerJSON[0].State.Status == "running"
// need to check for mounts, in case the builder-config is provided
// by default, we assume the configPath is correct
var configPathCorrect = true
if configPath != "" {
// if it is provided, we assume it is false until proven true
log.Debugf("checking if configPath %s is correct in container %s", configPath, name)
configPathCorrect = false
if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "exec", name, "cat", buildkitConfigPath); err == nil {
// sha256sum the config file to see if it matches the provided configPath
containerConfigFileHash := sha256.Sum256(b.Bytes())
log.Debugf("container %s has configPath %s with sha256sum %x", name, buildkitConfigPath, containerConfigFileHash)
configFileContents, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("unable to read buildkit config file %s: %v", configPath, err)
}
localConfigFileHash := sha256.Sum256(configFileContents)
log.Debugf("local %s has configPath %s with sha256sum %x", name, configPath, localConfigFileHash)
if bytes.Equal(containerConfigFileHash[:], localConfigFileHash[:]) {
log.Debugf("configPath %s in container %s matches local configPath %s", buildkitConfigPath, name, configPath)
configPathCorrect = true
} else {
log.Debugf("configPath %s in container %s does not match local configPath %s", buildkitConfigPath, name, configPath)
}
} else {
log.Debugf("could not read configPath %s from container %s, assuming it is not correct", buildkitConfigPath, name)
}
// now rewrite and copy over certs, if needed
//https://github.com/docker/buildx/blob/master/util/confutil/container.go#L27
}
switch {
case forceRestart:
@@ -306,6 +346,11 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
recreate = true
stop = isRunning
remove = true
case !configPathCorrect:
fmt.Printf("existing container has wrong configPath mount, restarting\n")
recreate = true
stop = isRunning
remove = true
case isRunning:
// if already running with the right image and permissions, just use it
fmt.Printf("using existing container %s\n", name)
@@ -326,18 +371,40 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
// if we made it here, we need to stop and remove the container, either because of a config mismatch,
// or because we received the CLI option
if stop {
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil {
return nil, fmt.Errorf("failed to stop existing container %s", name)
if cid == "" {
// we don't have a container ID, so we can't stop it
return nil, fmt.Errorf("unable to stop existing container %s, no ID found", name)
}
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", cid); err != nil {
// if we failed, do a retry; maybe it does not even exist anymore
time.Sleep(buildkitCheckInterval)
continue
}
}
if remove {
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil {
return nil, fmt.Errorf("failed to remove existing container %s", name)
if cid == "" {
// we don't have a container ID, so we can't remove it
return nil, fmt.Errorf("unable to remove existing container %s, no ID found", name)
}
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", cid); err != nil {
// mark the existing container as non-existent
cid = ""
// if we failed, do a retry; maybe it does not even exist anymore
time.Sleep(buildkitCheckInterval)
continue
}
}
if recreate {
// create the builder
args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"}
// this could be a single line, but it would be long. And it is easier to read when the
// docker command args, the image name, and the image args are all on separate lines.
args := []string{"--context", dockerContext, "container", "create", "--name", name, "--privileged"}
args = append(args, image)
args = append(args, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug")
if configPath != "" {
// set the config path explicitly
args = append(args, "--config", buildkitConfigPath)
}
msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext)
fmt.Println(msg)
if err := dr.command(nil, nil, io.Discard, args...); err != nil {
@@ -345,6 +412,22 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
time.Sleep(buildkitCheckInterval)
continue
}
// copy in the buildkit config file, if provided
if configPath != "" {
files, err := confutil.LoadConfigFiles(configPath)
if err != nil {
return nil, fmt.Errorf("failed to load buildkit config file %s: %v", configPath, err)
}
if err := dr.copyFilesToContainer(name, files); err != nil {
return nil, fmt.Errorf("failed to copy buildkit config file %s and certificates into container %s: %v", configPath, name, err)
}
}
// and now start the container
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil {
// if we failed, do a retry; maybe it does not even exist anymore
return nil, fmt.Errorf("failed to start newly created container %s: %v", name, err)
}
}
found = true
break
@@ -428,9 +511,9 @@ func (dr *dockerRunnerImpl) tag(ref, tag string) error {
return dr.command(nil, nil, nil, "image", "tag", ref, tag)
}
func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
// ensure we have a builder
client, err := dr.builder(ctx, dockerContext, builderImage, platform, restart)
client, err := dr.builder(ctx, dockerContext, builderImage, builderConfigPath, platform, restart)
if err != nil {
return fmt.Errorf("unable to ensure builder container: %v", err)
}
@@ -481,6 +564,7 @@ func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext,
attachable := []session.Attachable{}
localDirs := map[string]string{}
// Add SSH agent provider if needed
if len(imageBuildOpts.SSH) > 0 {
configs, err := build.ParseSSH(imageBuildOpts.SSH)
if err != nil {
@@ -501,8 +585,33 @@ func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext,
} else {
localDirs[dockerui.DefaultLocalNameDockerfile] = pkg
localDirs[dockerui.DefaultLocalNameContext] = pkg
}
// add credentials
var cf *configfile.ConfigFile
if len(imageBuildOpts.RegistryAuths) > 0 {
// if static ones were provided, use those
cf = configfile.New("custom")
// merge imageBuildOpts.RegistryAuths into dockercfg
for registry, auth := range imageBuildOpts.RegistryAuths {
// special case for docker.io
registryWithoutScheme := strings.TrimPrefix(registry, "https://")
registryWithoutScheme = strings.TrimPrefix(registryWithoutScheme, "http://")
if registryWithoutScheme == "docker.io" || registryWithoutScheme == "index.docker.io" || registryWithoutScheme == "registry-1.docker.io" {
registry = "https://index.docker.io/v1/"
}
cf.AuthConfigs[registry] = dockerconfigtypes.AuthConfig{
Username: auth.Username,
Password: auth.Password,
RegistryToken: auth.RegistryToken,
}
}
} else {
// Else use Docker authentication provider so BuildKit can use ~/.docker/config.json or OS-specific credential helpers.
cf = dockerconfig.LoadDefaultConfigFile(io.Discard)
}
attachable = append(attachable,
authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{ConfigFile: cf}),
)
solveOpts := buildkitClient.SolveOpt{
Frontend: "dockerfile.v0",
@@ -636,6 +745,33 @@ func (dr *dockerRunnerImpl) load(src io.Reader) error {
return dr.command(src, nil, nil, args...)
}
func (dr *dockerRunnerImpl) copyFilesToContainer(containerID string, files map[string][]byte) error {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for path, content := range files {
hdr := &tar.Header{
Name: path,
Mode: 0644,
Size: int64(len(content)),
ModTime: time.Now(),
Typeflag: tar.TypeReg,
}
if err := tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("write tar header: %w", err)
}
if _, err := tw.Write(content); err != nil {
return fmt.Errorf("write tar content: %w", err)
}
}
if err := tw.Close(); err != nil {
return fmt.Errorf("close tar: %w", err)
}
// Send the TAR archive to the container at /
return dr.command(&buf, os.Stdout, os.Stderr, "container", "cp", "-", containerID+":"+buildkitConfigDir)
}
func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil

View File

@@ -180,36 +180,36 @@ func (g git) commitTag(commit string) (string, error) {
}
func (g git) isDirty(pkg, commit string) (bool, error) {
// If it isn't HEAD it can't be dirty
// Only makes sense to check for HEAD
if commit != "HEAD" {
return false, nil
}
// Update cache, otherwise files which have an updated
// timestamp but no actual changes are marked as changes
// because `git diff-index` only uses the `lstat` result and
// not the actual file contents. Running `git update-index
// --refresh` updates the cache.
if err := g.command("update-index", "-q", "--refresh"); err != nil {
// 1. Check for changes in tracked files (without using update-index)
// --no-ext-diff disables any external diff tool
// --exit-code makes it return 1 if differences are found
err := g.command("diff", "--no-ext-diff", "--exit-code", "--quiet", commit, "--", pkg)
if err != nil {
if _, ok := err.(*exec.ExitError); ok {
// Changes found in tracked files
return true, nil
}
// Some actual failure
return false, err
}
// diff-index works pretty well, except that
err := g.command("diff-index", "--quiet", commit, "--", pkg)
// 2. Check for untracked files
_, err = g.commandStdout(nil, "ls-files", "--exclude-standard", "--others", "--error-unmatch", "--", pkg)
if err == nil {
// this returns an error if there are *no* untracked files, which is strange, but we can work with it
if _, err := g.commandStdout(nil, "ls-files", "--exclude-standard", "--others", "--error-unmatch", "--", pkg); err != nil {
return false, nil
}
// Untracked files found
return true, nil
}
switch err.(type) {
case *exec.ExitError:
// diff-index exits with an error if there are differences
return true, nil
default:
return false, err
if _, ok := err.(*exec.ExitError); ok {
// No untracked files — clean
return false, nil
}
// Unexpected error
return false, err
}
// goPkgVersion return a version that is compliant with go package versioning.

View File

@@ -93,14 +93,14 @@ func printVerbose(tw *tabwriter.Writer, du []*buildkitClient.UsageInfo) {
_ = tw.Flush()
}
func getClientForPlatform(ctx context.Context, buildersMap map[string]string, builderImage, platform string) (*buildkitClient.Client, error) {
func getClientForPlatform(ctx context.Context, buildersMap map[string]string, builderImage, builderConfigPath, platform string) (*buildkitClient.Client, error) {
p, err := platforms.Parse(platform)
if err != nil {
return nil, fmt.Errorf("failed to parse platform: %s", err)
}
dr := newDockerRunner(false)
builderName := getBuilderForPlatform(p.Architecture, buildersMap)
client, err := dr.builder(ctx, builderName, builderImage, platform, false)
client, err := dr.builder(ctx, builderName, builderImage, builderConfigPath, platform, false)
if err != nil {
return nil, fmt.Errorf("unable to ensure builder container: %v", err)
}
@@ -108,11 +108,11 @@ func getClientForPlatform(ctx context.Context, buildersMap map[string]string, bu
}
// DiskUsage of builder
func DiskUsage(buildersMap map[string]string, builderImage string, platformsToClean []string, verbose bool) error {
func DiskUsage(buildersMap map[string]string, builderImage, builderConfigPath string, platformsToClean []string, verbose bool) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for _, platform := range platformsToClean {
client, err := getClientForPlatform(ctx, buildersMap, builderImage, platform)
client, err := getClientForPlatform(ctx, buildersMap, builderImage, builderConfigPath, platform)
if err != nil {
return fmt.Errorf("cannot get client: %s", err)
}
@@ -143,12 +143,12 @@ func DiskUsage(buildersMap map[string]string, builderImage string, platformsToCl
}
// PruneBuilder clean build cache of builder
func PruneBuilder(buildersMap map[string]string, builderImage string, platformsToClean []string, verbose bool) error {
func PruneBuilder(buildersMap map[string]string, builderImage, builderConfigPath string, platformsToClean []string, verbose bool) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
total := int64(0)
for _, platform := range platformsToClean {
client, err := getClientForPlatform(ctx, buildersMap, builderImage, platform)
client, err := getClientForPlatform(ctx, buildersMap, builderImage, builderConfigPath, platform)
if err != nil {
return fmt.Errorf("cannot get client: %s", err)
}

View File

@@ -1,9 +1,16 @@
package spec
type ImageBuildOptions struct {
Labels map[string]string
BuildArgs map[string]*string
NetworkMode string
Dockerfile string
SSH []string
type RegistryAuth struct {
Username string
Password string
RegistryToken string // base64 encoded auth token
}
type ImageBuildOptions struct {
Labels map[string]string
BuildArgs map[string]*string
NetworkMode string
Dockerfile string
SSH []string
RegistryAuths map[string]RegistryAuth
}

View File

@@ -37,7 +37,7 @@ type CacheProvider interface {
ImageLoad(r io.Reader) ([]v1.Descriptor, error)
// DescriptorWrite writes a descriptor to the cache index; it validates that it has a name
// and replaces any existing one
DescriptorWrite(ref *reference.Spec, descriptors v1.Descriptor) error
DescriptorWrite(image string, descriptors v1.Descriptor) error
// Push an image along with a multi-arch index from local cache to remote registry.
// name is the name as referenced in the local cache, remoteName is the name to give it remotely.
// If remoteName is empty, it is the same as name.

View File

@@ -0,0 +1,9 @@
package util
import (
"os"
)
type FileLock struct {
file *os.File
}

View File

@@ -0,0 +1,19 @@
//go:build !unix
package util
// Lock opens the file (creating it if needed) and sets an exclusive lock.
// Returns a FileLock that can later be unlocked.
func Lock(path string) (*FileLock, error) {
return &FileLock{}, nil
}
// Unlock releases the lock and closes the file.
func (l *FileLock) Unlock() error {
return nil
}
// CheckLock attempts to detect if the file is locked by another process.
func CheckLock(path string) (locked bool, holderPID int, err error) {
return false, 0, nil
}

View File

@@ -0,0 +1,108 @@
//go:build unix
package util
import (
"fmt"
"io"
"os"
"golang.org/x/sys/unix"
)
// Lock opens the file (creating it if needed) and sets an exclusive lock.
// Returns a FileLock that can later be unlocked.
func Lock(path string) (*FileLock, error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return nil, fmt.Errorf("open file: %w", err)
}
flock := unix.Flock_t{
Type: unix.F_WRLCK,
Whence: int16(io.SeekStart),
Start: 0,
Len: 0,
}
if err := unix.FcntlFlock(f.Fd(), unix.F_SETLKW, &flock); err != nil {
_ = f.Close()
return nil, fmt.Errorf("set lock: %w", err)
}
return &FileLock{file: f}, nil
}
// Unlock releases the lock and closes the file.
func (l *FileLock) Unlock() error {
if l == nil || l.file == nil {
return fmt.Errorf("unlock: file handle is nil")
}
flock := unix.Flock_t{
Type: unix.F_UNLCK,
Whence: int16(io.SeekStart),
Start: 0,
Len: 0,
}
if err := unix.FcntlFlock(l.file.Fd(), unix.F_SETLKW, &flock); err != nil {
return fmt.Errorf("unlock: %w", err)
}
if err := l.file.Close(); err != nil {
return fmt.Errorf("close lock file: %w", err)
}
l.file = nil // Prevent further use of the file handle
return nil
}
// CheckLock attempts to detect if the file is locked by another process.
func CheckLock(path string) (locked bool, holderPID int, err error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return false, 0, fmt.Errorf("open file: %w", err)
}
defer func() {
_ = f.Close()
}()
check := unix.Flock_t{
Type: unix.F_WRLCK,
Whence: int16(io.SeekStart),
Start: 0,
Len: 0,
}
if err := unix.FcntlFlock(f.Fd(), unix.F_GETLK, &check); err != nil {
return false, 0, fmt.Errorf("get lock: %w", err)
}
if check.Type == unix.F_UNLCK {
return false, 0, nil
}
return true, int(check.Pid), nil
}
// WaitUnlocked waits until the file is unlocked by another process, and uses it for reading but not writing.
func WaitUnlocked(path string) error {
f, err := os.OpenFile(path, os.O_RDONLY, 0644)
if err != nil {
return fmt.Errorf("open file: %w", err)
}
defer func() {
_ = f.Close()
}()
flock := unix.Flock_t{
Type: unix.F_RDLCK,
Whence: int16(io.SeekStart),
Start: 0,
Len: 0,
}
if err := unix.FcntlFlock(f.Fd(), unix.F_SETLKW, &flock); err != nil {
_ = f.Close()
return fmt.Errorf("set lock: %w", err)
}
fileRef := &FileLock{file: f}
_ = fileRef.Unlock()
return nil
}

View File

@@ -0,0 +1,25 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
# IDEs
.idea/

View File

@@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Cenk Altı
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@@ -0,0 +1,30 @@
# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Coverage Status][coveralls image]][coveralls]
This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client].
[Exponential backoff][exponential backoff wiki]
is an algorithm that uses feedback to multiplicatively decrease the rate of some process,
in order to gradually find an acceptable rate.
The retries exponentially increase and stop increasing when a certain threshold is met.
## Usage
Import path is `github.com/cenkalti/backoff/v4`. Please note the version part at the end.
Use https://pkg.go.dev/github.com/cenkalti/backoff/v4 to view the documentation.
## Contributing
* I would like to keep this library as small as possible.
* Please don't send a PR without opening an issue and discussing it first.
* If proposed change is not a common use case, I will probably not accept it.
[godoc]: https://pkg.go.dev/github.com/cenkalti/backoff/v4
[godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png
[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master
[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master
[google-http-java-client]: https://github.com/google/google-http-java-client/blob/da1aa993e90285ec18579f1553339b00e19b3ab5/google-http-client/src/main/java/com/google/api/client/util/ExponentialBackOff.java
[exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff
[advanced example]: https://pkg.go.dev/github.com/cenkalti/backoff/v4?tab=doc#pkg-examples

View File

@@ -0,0 +1,66 @@
// Package backoff implements backoff algorithms for retrying operations.
//
// Use Retry function for retrying operations that may fail.
// If Retry does not meet your needs,
// copy/paste the function into your project and modify as you wish.
//
// There is also Ticker type similar to time.Ticker.
// You can use it if you need to work with channels.
//
// See Examples section below for usage examples.
package backoff
import "time"
// BackOff is a backoff policy for retrying an operation.
type BackOff interface {
// NextBackOff returns the duration to wait before retrying the operation,
// or backoff. Stop to indicate that no more retries should be made.
//
// Example usage:
//
// duration := backoff.NextBackOff();
// if (duration == backoff.Stop) {
// // Do not retry operation.
// } else {
// // Sleep for duration and retry operation.
// }
//
NextBackOff() time.Duration
// Reset to initial state.
Reset()
}
// Stop indicates that no more retries should be made for use in NextBackOff().
const Stop time.Duration = -1
// ZeroBackOff is a fixed backoff policy whose backoff time is always zero,
// meaning that the operation is retried immediately without waiting, indefinitely.
type ZeroBackOff struct{}
func (b *ZeroBackOff) Reset() {}
func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 }
// StopBackOff is a fixed backoff policy that always returns backoff.Stop for
// NextBackOff(), meaning that the operation should never be retried.
type StopBackOff struct{}
func (b *StopBackOff) Reset() {}
func (b *StopBackOff) NextBackOff() time.Duration { return Stop }
// ConstantBackOff is a backoff policy that always returns the same backoff delay.
// This is in contrast to an exponential backoff policy,
// which returns a delay that grows longer as you call NextBackOff() over and over again.
type ConstantBackOff struct {
Interval time.Duration
}
func (b *ConstantBackOff) Reset() {}
func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval }
func NewConstantBackOff(d time.Duration) *ConstantBackOff {
return &ConstantBackOff{Interval: d}
}

View File

@@ -0,0 +1,62 @@
package backoff
import (
"context"
"time"
)
// BackOffContext is a backoff policy that stops retrying after the context
// is canceled.
type BackOffContext interface { // nolint: golint
BackOff
Context() context.Context
}
type backOffContext struct {
BackOff
ctx context.Context
}
// WithContext returns a BackOffContext with context ctx
//
// ctx must not be nil
func WithContext(b BackOff, ctx context.Context) BackOffContext { // nolint: golint
if ctx == nil {
panic("nil context")
}
if b, ok := b.(*backOffContext); ok {
return &backOffContext{
BackOff: b.BackOff,
ctx: ctx,
}
}
return &backOffContext{
BackOff: b,
ctx: ctx,
}
}
func getContext(b BackOff) context.Context {
if cb, ok := b.(BackOffContext); ok {
return cb.Context()
}
if tb, ok := b.(*backOffTries); ok {
return getContext(tb.delegate)
}
return context.Background()
}
func (b *backOffContext) Context() context.Context {
return b.ctx
}
func (b *backOffContext) NextBackOff() time.Duration {
select {
case <-b.ctx.Done():
return Stop
default:
return b.BackOff.NextBackOff()
}
}

View File

@@ -0,0 +1,216 @@
package backoff
import (
"math/rand"
"time"
)
/*
ExponentialBackOff is a backoff implementation that increases the backoff
period for each retry attempt using a randomization function that grows exponentially.
NextBackOff() is calculated using the following formula:
randomized interval =
RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor])
In other words NextBackOff() will range between the randomization factor
percentage below and above the retry interval.
For example, given the following parameters:
RetryInterval = 2
RandomizationFactor = 0.5
Multiplier = 2
the actual backoff period used in the next retry attempt will range between 1 and 3 seconds,
multiplied by the exponential, that is, between 2 and 6 seconds.
Note: MaxInterval caps the RetryInterval and not the randomized interval.
If the time elapsed since an ExponentialBackOff instance is created goes past the
MaxElapsedTime, then the method NextBackOff() starts returning backoff.Stop.
The elapsed time can be reset by calling Reset().
Example: Given the following default arguments, for 10 tries the sequence will be,
and assuming we go over the MaxElapsedTime on the 10th try:
Request # RetryInterval (seconds) Randomized Interval (seconds)
1 0.5 [0.25, 0.75]
2 0.75 [0.375, 1.125]
3 1.125 [0.562, 1.687]
4 1.687 [0.8435, 2.53]
5 2.53 [1.265, 3.795]
6 3.795 [1.897, 5.692]
7 5.692 [2.846, 8.538]
8 8.538 [4.269, 12.807]
9 12.807 [6.403, 19.210]
10 19.210 backoff.Stop
Note: Implementation is not thread-safe.
*/
type ExponentialBackOff struct {
InitialInterval time.Duration
RandomizationFactor float64
Multiplier float64
MaxInterval time.Duration
// After MaxElapsedTime the ExponentialBackOff returns Stop.
// It never stops if MaxElapsedTime == 0.
MaxElapsedTime time.Duration
Stop time.Duration
Clock Clock
currentInterval time.Duration
startTime time.Time
}
// Clock is an interface that returns current time for BackOff.
type Clock interface {
Now() time.Time
}
// ExponentialBackOffOpts is a function type used to configure ExponentialBackOff options.
type ExponentialBackOffOpts func(*ExponentialBackOff)
// Default values for ExponentialBackOff.
const (
DefaultInitialInterval = 500 * time.Millisecond
DefaultRandomizationFactor = 0.5
DefaultMultiplier = 1.5
DefaultMaxInterval = 60 * time.Second
DefaultMaxElapsedTime = 15 * time.Minute
)
// NewExponentialBackOff creates an instance of ExponentialBackOff using default values.
func NewExponentialBackOff(opts ...ExponentialBackOffOpts) *ExponentialBackOff {
b := &ExponentialBackOff{
InitialInterval: DefaultInitialInterval,
RandomizationFactor: DefaultRandomizationFactor,
Multiplier: DefaultMultiplier,
MaxInterval: DefaultMaxInterval,
MaxElapsedTime: DefaultMaxElapsedTime,
Stop: Stop,
Clock: SystemClock,
}
for _, fn := range opts {
fn(b)
}
b.Reset()
return b
}
// WithInitialInterval sets the initial interval between retries.
func WithInitialInterval(duration time.Duration) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.InitialInterval = duration
}
}
// WithRandomizationFactor sets the randomization factor to add jitter to intervals.
func WithRandomizationFactor(randomizationFactor float64) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.RandomizationFactor = randomizationFactor
}
}
// WithMultiplier sets the multiplier for increasing the interval after each retry.
func WithMultiplier(multiplier float64) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.Multiplier = multiplier
}
}
// WithMaxInterval sets the maximum interval between retries.
func WithMaxInterval(duration time.Duration) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.MaxInterval = duration
}
}
// WithMaxElapsedTime sets the maximum total time for retries.
func WithMaxElapsedTime(duration time.Duration) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.MaxElapsedTime = duration
}
}
// WithRetryStopDuration sets the duration after which retries should stop.
func WithRetryStopDuration(duration time.Duration) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.Stop = duration
}
}
// WithClockProvider sets the clock used to measure time.
func WithClockProvider(clock Clock) ExponentialBackOffOpts {
return func(ebo *ExponentialBackOff) {
ebo.Clock = clock
}
}
type systemClock struct{}
func (t systemClock) Now() time.Time {
return time.Now()
}
// SystemClock implements Clock interface that uses time.Now().
var SystemClock = systemClock{}
// Reset the interval back to the initial retry interval and restarts the timer.
// Reset must be called before using b.
func (b *ExponentialBackOff) Reset() {
b.currentInterval = b.InitialInterval
b.startTime = b.Clock.Now()
}
// NextBackOff calculates the next backoff interval using the formula:
// Randomized interval = RetryInterval * (1 ± RandomizationFactor)
func (b *ExponentialBackOff) NextBackOff() time.Duration {
// Make sure we have not gone over the maximum elapsed time.
elapsed := b.GetElapsedTime()
next := getRandomValueFromInterval(b.RandomizationFactor, rand.Float64(), b.currentInterval)
b.incrementCurrentInterval()
if b.MaxElapsedTime != 0 && elapsed+next > b.MaxElapsedTime {
return b.Stop
}
return next
}
// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance
// is created and is reset when Reset() is called.
//
// The elapsed time is computed using time.Now().UnixNano(). It is
// safe to call even while the backoff policy is used by a running
// ticker.
func (b *ExponentialBackOff) GetElapsedTime() time.Duration {
return b.Clock.Now().Sub(b.startTime)
}
// Increments the current interval by multiplying it with the multiplier.
func (b *ExponentialBackOff) incrementCurrentInterval() {
// Check for overflow, if overflow is detected set the current interval to the max interval.
if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier {
b.currentInterval = b.MaxInterval
} else {
b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier)
}
}
// Returns a random value from the following interval:
// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval].
func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration {
if randomizationFactor == 0 {
return currentInterval // make sure no randomness is used when randomizationFactor is 0.
}
var delta = randomizationFactor * float64(currentInterval)
var minInterval = float64(currentInterval) - delta
var maxInterval = float64(currentInterval) + delta
// Get a random value from the range [minInterval, maxInterval].
// The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then
// we want a 33% chance for selecting either 1, 2 or 3.
return time.Duration(minInterval + (random * (maxInterval - minInterval + 1)))
}

View File

@@ -0,0 +1,146 @@
package backoff
import (
"errors"
"time"
)
// An OperationWithData is executing by RetryWithData() or RetryNotifyWithData().
// The operation will be retried using a backoff policy if it returns an error.
type OperationWithData[T any] func() (T, error)
// An Operation is executing by Retry() or RetryNotify().
// The operation will be retried using a backoff policy if it returns an error.
type Operation func() error
func (o Operation) withEmptyData() OperationWithData[struct{}] {
return func() (struct{}, error) {
return struct{}{}, o()
}
}
// Notify is a notify-on-error function. It receives an operation error and
// backoff delay if the operation failed (with an error).
//
// NOTE that if the backoff policy stated to stop retrying,
// the notify function isn't called.
type Notify func(error, time.Duration)
// Retry the operation o until it does not return error or BackOff stops.
// o is guaranteed to be run at least once.
//
// If o returns a *PermanentError, the operation is not retried, and the
// wrapped error is returned.
//
// Retry sleeps the goroutine for the duration returned by BackOff after a
// failed operation returns.
func Retry(o Operation, b BackOff) error {
return RetryNotify(o, b, nil)
}
// RetryWithData is like Retry but returns data in the response too.
func RetryWithData[T any](o OperationWithData[T], b BackOff) (T, error) {
return RetryNotifyWithData(o, b, nil)
}
// RetryNotify calls notify function with the error and wait duration
// for each failed attempt before sleep.
func RetryNotify(operation Operation, b BackOff, notify Notify) error {
return RetryNotifyWithTimer(operation, b, notify, nil)
}
// RetryNotifyWithData is like RetryNotify but returns data in the response too.
func RetryNotifyWithData[T any](operation OperationWithData[T], b BackOff, notify Notify) (T, error) {
return doRetryNotify(operation, b, notify, nil)
}
// RetryNotifyWithTimer calls notify function with the error and wait duration using the given Timer
// for each failed attempt before sleep.
// A default timer that uses system timer is used when nil is passed.
func RetryNotifyWithTimer(operation Operation, b BackOff, notify Notify, t Timer) error {
_, err := doRetryNotify(operation.withEmptyData(), b, notify, t)
return err
}
// RetryNotifyWithTimerAndData is like RetryNotifyWithTimer but returns data in the response too.
func RetryNotifyWithTimerAndData[T any](operation OperationWithData[T], b BackOff, notify Notify, t Timer) (T, error) {
return doRetryNotify(operation, b, notify, t)
}
func doRetryNotify[T any](operation OperationWithData[T], b BackOff, notify Notify, t Timer) (T, error) {
var (
err error
next time.Duration
res T
)
if t == nil {
t = &defaultTimer{}
}
defer func() {
t.Stop()
}()
ctx := getContext(b)
b.Reset()
for {
res, err = operation()
if err == nil {
return res, nil
}
var permanent *PermanentError
if errors.As(err, &permanent) {
return res, permanent.Err
}
if next = b.NextBackOff(); next == Stop {
if cerr := ctx.Err(); cerr != nil {
return res, cerr
}
return res, err
}
if notify != nil {
notify(err, next)
}
t.Start(next)
select {
case <-ctx.Done():
return res, ctx.Err()
case <-t.C():
}
}
}
// PermanentError signals that the operation should not be retried.
type PermanentError struct {
Err error
}
func (e *PermanentError) Error() string {
return e.Err.Error()
}
func (e *PermanentError) Unwrap() error {
return e.Err
}
func (e *PermanentError) Is(target error) bool {
_, ok := target.(*PermanentError)
return ok
}
// Permanent wraps the given err in a *PermanentError.
func Permanent(err error) error {
if err == nil {
return nil
}
return &PermanentError{
Err: err,
}
}

View File

@@ -0,0 +1,97 @@
package backoff
import (
"context"
"sync"
"time"
)
// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
//
// Ticks will continue to arrive when the previous operation is still running,
// so operations that take a while to fail could run in quick succession.
type Ticker struct {
C <-chan time.Time
c chan time.Time
b BackOff
ctx context.Context
timer Timer
stop chan struct{}
stopOnce sync.Once
}
// NewTicker returns a new Ticker containing a channel that will send
// the time at times specified by the BackOff argument. Ticker is
// guaranteed to tick at least once. The channel is closed when Stop
// method is called or BackOff stops. It is not safe to manipulate the
// provided backoff policy (notably calling NextBackOff or Reset)
// while the ticker is running.
func NewTicker(b BackOff) *Ticker {
return NewTickerWithTimer(b, &defaultTimer{})
}
// NewTickerWithTimer returns a new Ticker with a custom timer.
// A default timer that uses system timer is used when nil is passed.
func NewTickerWithTimer(b BackOff, timer Timer) *Ticker {
if timer == nil {
timer = &defaultTimer{}
}
c := make(chan time.Time)
t := &Ticker{
C: c,
c: c,
b: b,
ctx: getContext(b),
timer: timer,
stop: make(chan struct{}),
}
t.b.Reset()
go t.run()
return t
}
// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
t.stopOnce.Do(func() { close(t.stop) })
}
func (t *Ticker) run() {
c := t.c
defer close(c)
// Ticker is guaranteed to tick at least once.
afterC := t.send(time.Now())
for {
if afterC == nil {
return
}
select {
case tick := <-afterC:
afterC = t.send(tick)
case <-t.stop:
t.c = nil // Prevent future ticks from being sent to the channel.
return
case <-t.ctx.Done():
return
}
}
}
func (t *Ticker) send(tick time.Time) <-chan time.Time {
select {
case t.c <- tick:
case <-t.stop:
return nil
}
next := t.b.NextBackOff()
if next == Stop {
t.Stop()
return nil
}
t.timer.Start(next)
return t.timer.C()
}

View File

@@ -0,0 +1,35 @@
package backoff
import "time"
type Timer interface {
Start(duration time.Duration)
Stop()
C() <-chan time.Time
}
// defaultTimer implements Timer interface using time.Timer
type defaultTimer struct {
timer *time.Timer
}
// C returns the timers channel which receives the current time when the timer fires.
func (t *defaultTimer) C() <-chan time.Time {
return t.timer.C
}
// Start starts the timer to fire after the given duration
func (t *defaultTimer) Start(duration time.Duration) {
if t.timer == nil {
t.timer = time.NewTimer(duration)
} else {
t.timer.Reset(duration)
}
}
// Stop is called when the timer is not used anymore and resources may be freed.
func (t *defaultTimer) Stop() {
if t.timer != nil {
t.timer.Stop()
}
}

View File

@@ -0,0 +1,38 @@
package backoff
import "time"
/*
WithMaxRetries creates a wrapper around another BackOff, which will
return Stop if NextBackOff() has been called too many times since
the last time Reset() was called
Note: Implementation is not thread-safe.
*/
func WithMaxRetries(b BackOff, max uint64) BackOff {
return &backOffTries{delegate: b, maxTries: max}
}
type backOffTries struct {
delegate BackOff
maxTries uint64
numTries uint64
}
func (b *backOffTries) NextBackOff() time.Duration {
if b.maxTries == 0 {
return Stop
}
if b.maxTries > 0 {
if b.maxTries <= b.numTries {
return Stop
}
b.numTries++
}
return b.delegate.NextBackOff()
}
func (b *backOffTries) Reset() {
b.numTries = 0
b.delegate.Reset()
}

View File

@@ -0,0 +1,21 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devices
import "fmt"
var ErrNotSupported = fmt.Errorf("not supported")

View File

@@ -0,0 +1,75 @@
//go:build !windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devices
import (
"fmt"
"os"
"syscall"
"golang.org/x/sys/unix"
)
func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
sys, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo")
}
//nolint:unconvert
dev := uint64(sys.Rdev)
return uint64(unix.Major(dev)), uint64(unix.Minor(dev)), nil
}
// mknod provides a shortcut for syscall.Mknod
func Mknod(p string, mode os.FileMode, maj, min int) error {
var (
m = syscallMode(mode.Perm())
dev uint64
)
if mode&os.ModeDevice != 0 {
dev = unix.Mkdev(uint32(maj), uint32(min))
if mode&os.ModeCharDevice != 0 {
m |= unix.S_IFCHR
} else {
m |= unix.S_IFBLK
}
} else if mode&os.ModeNamedPipe != 0 {
m |= unix.S_IFIFO
}
return mknod(p, m, dev)
}
// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i os.FileMode) (o uint32) {
o |= uint32(i.Perm())
if i&os.ModeSetuid != 0 {
o |= unix.S_ISUID
}
if i&os.ModeSetgid != 0 {
o |= unix.S_ISGID
}
if i&os.ModeSticky != 0 {
o |= unix.S_ISVTX
}
return
}

View File

@@ -0,0 +1,26 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devices
import (
"fmt"
"os"
)
func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
return 0, 0, fmt.Errorf("cannot get device info on windows: %w", ErrNotSupported)
}

View File

@@ -0,0 +1,25 @@
//go:build freebsd || dragonfly
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devices
import "golang.org/x/sys/unix"
func mknod(path string, mode uint32, dev uint64) (err error) {
return unix.Mknod(path, mode, dev)
}

View File

@@ -0,0 +1,25 @@
//go:build !(freebsd || windows)
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devices
import "golang.org/x/sys/unix"
func mknod(path string, mode uint32, dev uint64) (err error) {
return unix.Mknod(path, mode, int(dev))
}

View File

@@ -0,0 +1,211 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"io"
"os"
"path/filepath"
"github.com/containerd/log"
)
// XAttrErrorHandler transform a non-nil xattr error.
// Return nil to ignore an error.
// xattrKey can be empty for listxattr operation.
type XAttrErrorHandler func(dst, src, xattrKey string, err error) error
type copyDirOpts struct {
xeh XAttrErrorHandler
// xex contains a set of xattrs to exclude when copying
xex map[string]struct{}
}
type CopyDirOpt func(*copyDirOpts) error
// WithXAttrErrorHandler allows specifying XAttrErrorHandler
// If nil XAttrErrorHandler is specified (default), CopyDir stops
// on a non-nil xattr error.
func WithXAttrErrorHandler(xeh XAttrErrorHandler) CopyDirOpt {
return func(o *copyDirOpts) error {
o.xeh = xeh
return nil
}
}
// WithAllowXAttrErrors allows ignoring xattr errors.
func WithAllowXAttrErrors() CopyDirOpt {
xeh := func(dst, src, xattrKey string, err error) error {
return nil
}
return WithXAttrErrorHandler(xeh)
}
// WithXAttrExclude allows for exclusion of specified xattr during CopyDir operation.
func WithXAttrExclude(keys ...string) CopyDirOpt {
return func(o *copyDirOpts) error {
if o.xex == nil {
o.xex = make(map[string]struct{}, len(keys))
}
for _, key := range keys {
o.xex[key] = struct{}{}
}
return nil
}
}
// CopyDir copies the directory from src to dst.
// Most efficient copy of files is attempted.
func CopyDir(dst, src string, opts ...CopyDirOpt) error {
var o copyDirOpts
for _, opt := range opts {
if err := opt(&o); err != nil {
return err
}
}
inodes := map[uint64]string{}
return copyDirectory(dst, src, inodes, &o)
}
func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) error {
stat, err := os.Stat(src)
if err != nil {
return fmt.Errorf("failed to stat %s: %w", src, err)
}
if !stat.IsDir() {
return fmt.Errorf("source %s is not directory", src)
}
if st, err := os.Stat(dst); err != nil {
if err := os.Mkdir(dst, stat.Mode()); err != nil {
return fmt.Errorf("failed to mkdir %s: %w", dst, err)
}
} else if !st.IsDir() {
return fmt.Errorf("cannot copy to non-directory: %s", dst)
} else {
if err := os.Chmod(dst, stat.Mode()); err != nil {
return fmt.Errorf("failed to chmod on %s: %w", dst, err)
}
}
if err := copyFileInfo(stat, src, dst); err != nil {
return fmt.Errorf("failed to copy file info for %s: %w", dst, err)
}
if err := copyXAttrs(dst, src, o.xex, o.xeh); err != nil {
return fmt.Errorf("failed to copy xattrs: %w", err)
}
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
dr := &dirReader{f: f}
handleEntry := func(entry os.DirEntry) error {
source := filepath.Join(src, entry.Name())
target := filepath.Join(dst, entry.Name())
fileInfo, err := entry.Info()
if err != nil {
return fmt.Errorf("failed to get file info for %s: %w", entry.Name(), err)
}
switch {
case entry.IsDir():
if err := copyDirectory(target, source, inodes, o); err != nil {
return err
}
return nil
case (fileInfo.Mode() & os.ModeType) == 0:
link, err := getLinkSource(target, fileInfo, inodes)
if err != nil {
return fmt.Errorf("failed to get hardlink: %w", err)
}
if link != "" {
if err := os.Link(link, target); err != nil {
return fmt.Errorf("failed to create hard link: %w", err)
}
} else if err := CopyFile(target, source); err != nil {
return fmt.Errorf("failed to copy files: %w", err)
}
case (fileInfo.Mode() & os.ModeSymlink) == os.ModeSymlink:
link, err := os.Readlink(source)
if err != nil {
return fmt.Errorf("failed to read link: %s: %w", source, err)
}
if err := os.Symlink(link, target); err != nil {
return fmt.Errorf("failed to create symlink: %s: %w", target, err)
}
case (fileInfo.Mode() & os.ModeDevice) == os.ModeDevice,
(fileInfo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe,
(fileInfo.Mode() & os.ModeSocket) == os.ModeSocket:
if err := copyIrregular(target, fileInfo); err != nil {
return fmt.Errorf("failed to create irregular file: %w", err)
}
default:
log.L.Warnf("unsupported mode: %s: %s", source, fileInfo.Mode())
return nil
}
if err := copyFileInfo(fileInfo, source, target); err != nil {
return fmt.Errorf("failed to copy file info: %w", err)
}
if err := copyXAttrs(target, source, o.xex, o.xeh); err != nil {
return fmt.Errorf("failed to copy xattrs: %w", err)
}
return nil
}
for {
entry := dr.Next()
if entry == nil {
break
}
if err := handleEntry(entry); err != nil {
return err
}
}
return dr.Err()
}
// CopyFile copies the source file to the target.
// The most efficient means of copying is used for the platform.
func CopyFile(target, source string) error {
return copyFile(target, source)
}
func openAndCopyFile(target, source string) error {
src, err := os.Open(source)
if err != nil {
return fmt.Errorf("failed to open source %s: %w", source, err)
}
defer src.Close()
tgt, err := os.Create(target)
if err != nil {
return fmt.Errorf("failed to open target %s: %w", target, err)
}
defer tgt.Close()
_, err = io.Copy(tgt, src)
return err
}

View File

@@ -0,0 +1,35 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"fmt"
"golang.org/x/sys/unix"
)
func copyFile(target, source string) error {
if err := unix.Clonefile(source, target, unix.CLONE_NOFOLLOW); err != nil {
if !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EXDEV) {
return fmt.Errorf("clonefile failed: %w", err)
}
return openAndCopyFile(target, source)
}
return nil
}

View File

@@ -0,0 +1,36 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"os"
"syscall"
)
// copyIrregular covers devices, pipes, and sockets
func copyIrregular(dst string, fi os.FileInfo) error {
st, ok := fi.Sys().(*syscall.Stat_t) // not *unix.Stat_t
if !ok {
return fmt.Errorf("unsupported stat type: %s: %v", dst, fi.Mode())
}
var rDev uint64 // uint64 on FreeBSD, int on other unixen
if fi.Mode()&os.ModeDevice == os.ModeDevice {
rDev = st.Rdev
}
return syscall.Mknod(dst, uint32(st.Mode), rDev)
}

View File

@@ -0,0 +1,39 @@
//go:build !windows && !freebsd
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"os"
"syscall"
)
// copyIrregular covers devices, pipes, and sockets
func copyIrregular(dst string, fi os.FileInfo) error {
st, ok := fi.Sys().(*syscall.Stat_t) // not *unix.Stat_t
if !ok {
return fmt.Errorf("unsupported stat type: %s: %v", dst, fi.Mode())
}
var rDev int
if fi.Mode()&os.ModeDevice == os.ModeDevice {
rDev = int(st.Rdev)
}
//nolint:unconvert
return syscall.Mknod(dst, uint32(st.Mode), rDev)
}

View File

@@ -0,0 +1,103 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"fmt"
"os"
"syscall"
"github.com/containerd/continuity/sysx"
"golang.org/x/sys/unix"
)
func copyFileInfo(fi os.FileInfo, src, name string) error {
st := fi.Sys().(*syscall.Stat_t)
if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
if os.IsPermission(err) {
// Normally if uid/gid are the same this would be a no-op, but some
// filesystems may still return EPERM... for instance NFS does this.
// In such a case, this is not an error.
if dstStat, err2 := os.Lstat(name); err2 == nil {
st2 := dstStat.Sys().(*syscall.Stat_t)
if st.Uid == st2.Uid && st.Gid == st2.Gid {
err = nil
}
}
}
if err != nil {
return fmt.Errorf("failed to chown %s: %w", name, err)
}
}
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
if err := os.Chmod(name, fi.Mode()); err != nil {
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
}
timespec := []unix.Timespec{
unix.NsecToTimespec(syscall.TimespecToNsec(StatAtime(st))),
unix.NsecToTimespec(syscall.TimespecToNsec(StatMtime(st))),
}
if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
return fmt.Errorf("failed to utime %s: %w", name, err)
}
return nil
}
func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
xattrKeys, err := sysx.LListxattr(src)
if err != nil {
if errors.Is(err, unix.ENOTSUP) {
return nil
}
e := fmt.Errorf("failed to list xattrs on %s: %w", src, err)
if errorHandler != nil {
e = errorHandler(dst, src, "", e)
}
return e
}
for _, xattr := range xattrKeys {
if _, exclude := excludes[xattr]; exclude {
continue
}
data, err := sysx.LGetxattr(src, xattr)
if err != nil {
e := fmt.Errorf("failed to get xattr %q on %s: %w", xattr, src, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
}
}
return e
}
if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
e := fmt.Errorf("failed to set xattr %q on %s: %w", xattr, dst, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
}
}
return e
}
}
return nil
}

View File

@@ -0,0 +1,21 @@
//go:build !darwin
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
var copyFile = openAndCopyFile

View File

@@ -0,0 +1,106 @@
//go:build darwin || freebsd || openbsd || netbsd || dragonfly || solaris
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"fmt"
"os"
"runtime"
"syscall"
"github.com/containerd/continuity/sysx"
"golang.org/x/sys/unix"
)
func copyFileInfo(fi os.FileInfo, src, name string) error {
st := fi.Sys().(*syscall.Stat_t)
if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
if os.IsPermission(err) {
// Normally if uid/gid are the same this would be a no-op, but some
// filesystems may still return EPERM... for instance NFS does this.
// In such a case, this is not an error.
if dstStat, err2 := os.Lstat(name); err2 == nil {
st2 := dstStat.Sys().(*syscall.Stat_t)
if st.Uid == st2.Uid && st.Gid == st2.Gid {
err = nil
}
}
}
if err != nil {
return fmt.Errorf("failed to chown %s: %w", name, err)
}
}
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
if err := os.Chmod(name, fi.Mode()); err != nil {
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
}
if err := utimesNano(name, StatAtime(st), StatMtime(st)); err != nil {
return fmt.Errorf("failed to utime %s: %w", name, err)
}
return nil
}
func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
xattrKeys, err := sysx.LListxattr(src)
if err != nil {
if os.IsPermission(err) && runtime.GOOS == "darwin" {
// On darwin, character devices do not permit listing xattrs
return nil
}
if errors.Is(err, unix.ENOTSUP) {
return nil
}
e := fmt.Errorf("failed to list xattrs on %s: %w", src, err)
if errorHandler != nil {
e = errorHandler(dst, src, "", e)
}
return e
}
for _, xattr := range xattrKeys {
if _, exclude := excludes[xattr]; exclude {
continue
}
data, err := sysx.LGetxattr(src, xattr)
if err != nil {
e := fmt.Errorf("failed to get xattr %q on %s: %w", xattr, src, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
}
}
return e
}
if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
e := fmt.Errorf("failed to set xattr %q on %s: %w", xattr, dst, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
}
}
return e
}
}
return nil
}

View File

@@ -0,0 +1,80 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"fmt"
"os"
winio "github.com/Microsoft/go-winio"
"golang.org/x/sys/windows"
)
const (
seTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege"
)
func copyFileInfo(fi os.FileInfo, src, name string) error {
if err := os.Chmod(name, fi.Mode()); err != nil {
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
// Copy file ownership and ACL
// We need SeRestorePrivilege and SeTakeOwnershipPrivilege in order
// to restore security info on a file, especially if we're trying to
// apply security info which includes SIDs not necessarily present on
// the host.
privileges := []string{winio.SeRestorePrivilege, seTakeOwnershipPrivilege}
if err := winio.EnableProcessPrivileges(privileges); err != nil {
return err
}
defer winio.DisableProcessPrivileges(privileges)
secInfo, err := windows.GetNamedSecurityInfo(
src, windows.SE_FILE_OBJECT,
windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION)
if err != nil {
return err
}
dacl, _, err := secInfo.DACL()
if err != nil {
return err
}
sid, _, err := secInfo.Owner()
if err != nil {
return err
}
if err := windows.SetNamedSecurityInfo(
name, windows.SE_FILE_OBJECT,
windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION,
sid, nil, dacl, nil); err != nil {
return err
}
return nil
}
func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
return nil
}
func copyIrregular(dst string, fi os.FileInfo) error {
return errors.New("irregular copy not supported")
}

View File

@@ -0,0 +1,365 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"github.com/containerd/log"
"golang.org/x/sync/errgroup"
)
// ChangeKind is the type of modification that
// a change is making.
type ChangeKind int
const (
// ChangeKindUnmodified represents an unmodified
// file
ChangeKindUnmodified = iota
// ChangeKindAdd represents an addition of
// a file
ChangeKindAdd
// ChangeKindModify represents a change to
// an existing file
ChangeKindModify
// ChangeKindDelete represents a delete of
// a file
ChangeKindDelete
)
func (k ChangeKind) String() string {
switch k {
case ChangeKindUnmodified:
return "unmodified"
case ChangeKindAdd:
return "add"
case ChangeKindModify:
return "modify"
case ChangeKindDelete:
return "delete"
default:
return ""
}
}
// Change represents single change between a diff and its parent.
type Change struct {
Kind ChangeKind
Path string
}
// ChangeFunc is the type of function called for each change
// computed during a directory changes calculation.
type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error
// Changes computes changes between two directories calling the
// given change function for each computed change. The first
// directory is intended to the base directory and second
// directory the changed directory.
//
// The change callback is called by the order of path names and
// should be appliable in that order.
//
// Due to this apply ordering, the following is true
// - Removed directory trees only create a single change for the root
// directory removed. Remaining changes are implied.
// - A directory which is modified to become a file will not have
// delete entries for sub-path items, their removal is implied
// by the removal of the parent directory.
//
// Opaque directories will not be treated specially and each file
// removed from the base directory will show up as a removal.
//
// File content comparisons will be done on files which have timestamps
// which may have been truncated. If either of the files being compared
// has a zero value nanosecond value, each byte will be compared for
// differences. If 2 files have the same seconds value but different
// nanosecond values where one of those values is zero, the files will
// be considered unchanged if the content is the same. This behavior
// is to account for timestamp truncation during archiving.
func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error {
if a == "" {
log.G(ctx).Debugf("Using single walk diff for %s", b)
return addDirChanges(ctx, changeFn, b)
}
log.G(ctx).Debugf("Using double walk diff for %s from %s", b, a)
return doubleWalkDiff(ctx, changeFn, a, b)
}
func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error {
return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// Rebase path
path, err = filepath.Rel(root, path)
if err != nil {
return err
}
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == string(os.PathSeparator) {
return nil
}
return changeFn(ChangeKindAdd, path, f, nil)
})
}
// DiffChangeSource is the source of diff directory.
type DiffSource int
const (
// DiffSourceOverlayFS indicates that a diff directory is from
// OverlayFS.
DiffSourceOverlayFS DiffSource = iota
)
// diffDirOptions is used when the diff can be directly calculated from
// a diff directory to its base, without walking both trees.
type diffDirOptions struct {
skipChange func(string, os.FileInfo) (bool, error)
deleteChange func(string, string, os.FileInfo, ChangeFunc) (bool, error)
}
// DiffDirChanges walks the diff directory and compares changes against the base.
//
// NOTE: If all the children of a dir are removed, or that dir are recreated
// after remove, we will mark non-existing `.wh..opq` file as deleted. It's
// unlikely to create explicit whiteout files for all the children and all
// descendants. And based on OCI spec, it's not possible to create a file or
// dir with a name beginning with `.wh.`. So, after `.wh..opq` file has been
// deleted, the ChangeFunc, the receiver will add whiteout prefix to create a
// opaque whiteout `.wh..wh..opq`.
//
// REF: https://github.com/opencontainers/image-spec/blob/v1.0/layer.md#whiteouts
func DiffDirChanges(ctx context.Context, baseDir, diffDir string, source DiffSource, changeFn ChangeFunc) error {
var o *diffDirOptions
switch source {
case DiffSourceOverlayFS:
o = &diffDirOptions{
deleteChange: overlayFSWhiteoutConvert,
}
default:
return errors.New("unknown diff change source")
}
changedDirs := make(map[string]struct{})
return filepath.Walk(diffDir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// Rebase path
path, err = filepath.Rel(diffDir, path)
if err != nil {
return err
}
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == string(os.PathSeparator) {
return nil
}
if o.skipChange != nil {
if skip, err := o.skipChange(path, f); skip {
return err
}
}
var kind ChangeKind
deletedFile := false
if o.deleteChange != nil {
deletedFile, err = o.deleteChange(diffDir, path, f, changeFn)
if err != nil {
return err
}
_, err = os.Stat(filepath.Join(baseDir, path))
if err != nil {
if !os.IsNotExist(err) {
return err
}
deletedFile = false
}
}
// Find out what kind of modification happened
if deletedFile {
kind = ChangeKindDelete
} else {
// Otherwise, the file was added
kind = ChangeKindAdd
// ...Unless it already existed in a baseDir, in which case, it's a modification
stat, err := os.Stat(filepath.Join(baseDir, path))
if err != nil && !os.IsNotExist(err) {
return err
}
if err == nil {
// The file existed in the baseDir, so that's a modification
// However, if it's a directory, maybe it wasn't actually modified.
// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
if stat.IsDir() && f.IsDir() {
if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
// Both directories are the same, don't record the change
return nil
}
}
kind = ChangeKindModify
}
}
// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
// This block is here to ensure the change is recorded even if the
// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
// Check https://github.com/docker/docker/pull/13590 for details.
if f.IsDir() {
changedDirs[path] = struct{}{}
}
if kind == ChangeKindAdd || kind == ChangeKindDelete {
parent := filepath.Dir(path)
if _, ok := changedDirs[parent]; !ok && parent != "/" {
pi, err := os.Stat(filepath.Join(diffDir, parent))
if err := changeFn(ChangeKindModify, parent, pi, err); err != nil {
return err
}
changedDirs[parent] = struct{}{}
}
}
if kind == ChangeKindDelete {
f = nil
}
return changeFn(kind, path, f, nil)
})
}
// doubleWalkDiff walks both directories to create a diff
func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) {
g, ctx := errgroup.WithContext(ctx)
var (
c1 = make(chan *currentPath)
c2 = make(chan *currentPath)
f1, f2 *currentPath
rmdir string
)
g.Go(func() error {
defer close(c1)
return pathWalk(ctx, a, c1)
})
g.Go(func() error {
defer close(c2)
return pathWalk(ctx, b, c2)
})
g.Go(func() error {
for c1 != nil || c2 != nil {
if f1 == nil && c1 != nil {
f1, err = nextPath(ctx, c1)
if err != nil {
return err
}
if f1 == nil {
c1 = nil
}
}
if f2 == nil && c2 != nil {
f2, err = nextPath(ctx, c2)
if err != nil {
return err
}
if f2 == nil {
c2 = nil
}
}
if f1 == nil && f2 == nil {
continue
}
var f os.FileInfo
k, p := pathChange(f1, f2)
switch k {
case ChangeKindAdd:
if rmdir != "" {
rmdir = ""
}
f = f2.f
f2 = nil
case ChangeKindDelete:
// Check if this file is already removed by being
// under of a removed directory
if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
f1 = nil
continue
} else if f1.f.IsDir() {
rmdir = f1.path + string(os.PathSeparator)
} else if rmdir != "" {
rmdir = ""
}
f1 = nil
case ChangeKindModify:
same, err := sameFile(f1, f2)
if err != nil {
return err
}
if f1.f.IsDir() && !f2.f.IsDir() {
rmdir = f1.path + string(os.PathSeparator)
} else if rmdir != "" {
rmdir = ""
}
f = f2.f
f1 = nil
f2 = nil
if same {
if !isLinked(f) {
continue
}
k = ChangeKindUnmodified
}
}
if err := changeFn(k, p, f, nil); err != nil {
return err
}
}
return nil
})
return g.Wait()
}

View File

@@ -0,0 +1,101 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
"github.com/containerd/continuity/devices"
"github.com/containerd/continuity/sysx"
"golang.org/x/sys/unix"
)
const (
// whiteoutPrefix prefix means file is a whiteout. If this is followed
// by a filename this means that file has been removed from the base
// layer.
//
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
whiteoutPrefix = ".wh."
)
// overlayFSWhiteoutConvert detects whiteouts and opaque directories.
//
// It returns deleted indicator if the file is a character device with 0/0
// device number. And call changeFn with ChangeKindDelete for opaque
// directories.
//
// Check: https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
func overlayFSWhiteoutConvert(diffDir, path string, f os.FileInfo, changeFn ChangeFunc) (deleted bool, _ error) {
if f.Mode()&os.ModeCharDevice != 0 {
if _, ok := f.Sys().(*syscall.Stat_t); !ok {
return false, nil
}
maj, min, err := devices.DeviceInfo(f)
if err != nil {
return false, err
}
return (maj == 0 && min == 0), nil
}
if f.IsDir() {
originalPath := filepath.Join(diffDir, path)
opaque, err := getOpaqueValue(originalPath)
if err != nil {
if errors.Is(err, unix.ENODATA) {
return false, nil
}
return false, err
}
if len(opaque) == 1 && opaque[0] == 'y' {
opaqueDirPath := filepath.Join(path, whiteoutPrefix+".opq")
return false, changeFn(ChangeKindDelete, opaqueDirPath, nil, nil)
}
}
return false, nil
}
// getOpaqueValue returns opaque value for a given file.
func getOpaqueValue(filePath string) ([]byte, error) {
for _, xattr := range []string{
"trusted.overlay.opaque",
// TODO(fuweid):
//
// user.overlay.* is available since 5.11. We should check
// kernel version before read.
//
// REF: https://github.com/torvalds/linux/commit/2d2f2d7322ff43e0fe92bf8cccdc0b09449bf2e1
"user.overlay.opaque",
} {
opaque, err := sysx.LGetxattr(filePath, xattr)
if err != nil {
if errors.Is(err, unix.ENODATA) || errors.Is(err, unix.ENOTSUP) {
continue
}
return nil, fmt.Errorf("failed to retrieve %s attr: %w", xattr, err)
}
return opaque, nil
}
return nil, unix.ENODATA
}

View File

@@ -0,0 +1,28 @@
//go:build !linux
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"os"
)
func overlayFSWhiteoutConvert(string, string, os.FileInfo, ChangeFunc) (bool, error) {
return false, errors.New("unsupported")
}

View File

@@ -0,0 +1,64 @@
//go:build !windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"bytes"
"fmt"
"os"
"syscall"
"github.com/containerd/continuity/sysx"
)
// compareSysStat returns whether the stats are equivalent,
// whether the files are considered the same file, and
// an error
func compareSysStat(s1, s2 interface{}) (bool, error) {
ls1, ok := s1.(*syscall.Stat_t)
if !ok {
return false, nil
}
ls2, ok := s2.(*syscall.Stat_t)
if !ok {
return false, nil
}
return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil
}
func compareCapabilities(p1, p2 string) (bool, error) {
c1, err := sysx.LGetxattr(p1, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, fmt.Errorf("failed to get xattr for %s: %w", p1, err)
}
c2, err := sysx.LGetxattr(p2, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, fmt.Errorf("failed to get xattr for %s: %w", p2, err)
}
return bytes.Equal(c1, c2), nil
}
func isLinked(f os.FileInfo) bool {
s, ok := f.Sys().(*syscall.Stat_t)
if !ok {
return false
}
return !f.IsDir() && s.Nlink > 1
}

View File

@@ -0,0 +1,44 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"os"
"golang.org/x/sys/windows"
)
func compareSysStat(s1, s2 interface{}) (bool, error) {
f1, ok := s1.(windows.Win32FileAttributeData)
if !ok {
return false, nil
}
f2, ok := s2.(windows.Win32FileAttributeData)
if !ok {
return false, nil
}
return f1.FileAttributes == f2.FileAttributes, nil
}
func compareCapabilities(p1, p2 string) (bool, error) {
// TODO: Use windows equivalent
return true, nil
}
func isLinked(os.FileInfo) bool {
return false
}

View File

@@ -0,0 +1,53 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"io"
"os"
)
type dirReader struct {
buf []os.DirEntry
f *os.File
err error
}
func (r *dirReader) Next() os.DirEntry {
if len(r.buf) == 0 {
infos, err := r.f.ReadDir(32)
if err != nil {
if err != io.EOF {
r.err = err
}
return nil
}
r.buf = infos
}
if len(r.buf) == 0 {
return nil
}
out := r.buf[0]
r.buf[0] = nil
r.buf = r.buf[1:]
return out
}
func (r *dirReader) Err() error {
return r.err
}

View File

@@ -0,0 +1,102 @@
//go:build linux
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"os"
"syscall"
"unsafe"
)
func locateDummyIfEmpty(path string) (string, error) {
children, err := os.ReadDir(path)
if err != nil {
return "", err
}
if len(children) != 0 {
return "", nil
}
dummyFile, err := os.CreateTemp(path, "fsutils-dummy")
if err != nil {
return "", err
}
name := dummyFile.Name()
err = dummyFile.Close()
return name, err
}
// SupportsDType returns whether the filesystem mounted on path supports d_type
func SupportsDType(path string) (bool, error) {
// locate dummy so that we have at least one dirent
dummy, err := locateDummyIfEmpty(path)
if err != nil {
return false, err
}
if dummy != "" {
defer os.Remove(dummy)
}
visited := 0
supportsDType := true
fn := func(ent *syscall.Dirent) bool {
visited++
if ent.Type == syscall.DT_UNKNOWN {
supportsDType = false
// stop iteration
return true
}
// continue iteration
return false
}
if err = iterateReadDir(path, fn); err != nil {
return false, err
}
if visited == 0 {
return false, fmt.Errorf("did not hit any dirent during iteration %s", path)
}
return supportsDType, nil
}
func iterateReadDir(path string, fn func(*syscall.Dirent) bool) error {
d, err := os.Open(path)
if err != nil {
return err
}
defer d.Close()
fd := int(d.Fd())
buf := make([]byte, 4096)
for {
nbytes, err := syscall.ReadDirent(fd, buf)
if err != nil {
return err
}
if nbytes == 0 {
break
}
for off := 0; off < nbytes; {
ent := (*syscall.Dirent)(unsafe.Pointer(&buf[off]))
if stop := fn(ent); stop {
return nil
}
off += int(ent.Reclen)
}
}
return nil
}

View File

@@ -0,0 +1,38 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import "context"
// Usage of disk information
type Usage struct {
Inodes int64
Size int64
}
// DiskUsage counts the number of inodes and disk usage for the resources under
// path.
func DiskUsage(ctx context.Context, roots ...string) (Usage, error) {
return diskUsage(ctx, roots...)
}
// DiffUsage counts the numbers of inodes and disk usage in the
// diff between the 2 directories. The first path is intended
// as the base directory and the second as the changed directory.
func DiffUsage(ctx context.Context, a, b string) (Usage, error) {
return diffUsage(ctx, a, b)
}

View File

@@ -0,0 +1,118 @@
//go:build !windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"context"
"os"
"path/filepath"
"syscall"
)
// blocksUnitSize is the unit used by `st_blocks` in `stat` in bytes.
// See https://man7.org/linux/man-pages/man2/stat.2.html
//
// st_blocks
// This field indicates the number of blocks allocated to the
// file, in 512-byte units. (This may be smaller than
// st_size/512 when the file has holes.)
const blocksUnitSize = 512
type inode struct {
// TODO(stevvooe): Can probably reduce memory usage by not tracking
// device, but we can leave this right for now.
dev, ino uint64
}
func newInode(stat *syscall.Stat_t) inode {
return inode{
dev: uint64(stat.Dev), //nolint: unconvert // dev is uint32 on darwin/bsd, uint64 on linux/solaris/freebsd
ino: uint64(stat.Ino), //nolint: unconvert // ino is uint32 on bsd, uint64 on darwin/linux/solaris/freebsd
}
}
func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
var (
size int64
inodes = map[inode]struct{}{} // expensive!
)
for _, root := range roots {
if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
stat := fi.Sys().(*syscall.Stat_t)
inoKey := newInode(stat)
if _, ok := inodes[inoKey]; !ok {
inodes[inoKey] = struct{}{}
size += stat.Blocks * blocksUnitSize
}
return nil
}); err != nil {
return Usage{}, err
}
}
return Usage{
Inodes: int64(len(inodes)),
Size: size,
}, nil
}
func diffUsage(ctx context.Context, a, b string) (Usage, error) {
var (
size int64
inodes = map[inode]struct{}{} // expensive!
)
if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if kind == ChangeKindAdd || kind == ChangeKindModify {
stat := fi.Sys().(*syscall.Stat_t)
inoKey := newInode(stat)
if _, ok := inodes[inoKey]; !ok {
inodes[inoKey] = struct{}{}
size += stat.Blocks * blocksUnitSize
}
return nil
}
return nil
}); err != nil {
return Usage{}, err
}
return Usage{
Inodes: int64(len(inodes)),
Size: size,
}, nil
}

View File

@@ -0,0 +1,78 @@
//go:build windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"context"
"os"
"path/filepath"
)
func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
var size int64
// TODO(stevvooe): Support inodes (or equivalent) for windows.
for _, root := range roots {
if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
size += fi.Size()
return nil
}); err != nil {
return Usage{}, err
}
}
return Usage{
Size: size,
}, nil
}
func diffUsage(ctx context.Context, a, b string) (Usage, error) {
var size int64
if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if kind == ChangeKindAdd || kind == ChangeKindModify {
size += fi.Size()
return nil
}
return nil
}); err != nil {
return Usage{}, err
}
return Usage{
Size: size,
}, nil
}

View File

@@ -0,0 +1,43 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import "os"
// GetLinkInfo returns an identifier representing the node a hardlink is pointing
// to. If the file is not hard linked then 0 will be returned.
func GetLinkInfo(fi os.FileInfo) (uint64, bool) {
return getLinkInfo(fi)
}
// getLinkSource returns a path for the given name and
// file info to its link source in the provided inode
// map. If the given file name is not in the map and
// has other links, it is added to the inode map
// to be a source for other link locations.
func getLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) {
inode, isHardlink := getLinkInfo(fi)
if !isHardlink {
return "", nil
}
path, ok := inodes[inode]
if !ok {
inodes[inode] = name
}
return path, nil
}

View File

@@ -0,0 +1,33 @@
//go:build !windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"os"
"syscall"
)
func getLinkInfo(fi os.FileInfo) (uint64, bool) {
s, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return 0, false
}
return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 //nolint: unconvert // ino is uint32 on bsd, uint64 on darwin/linux/solaris
}

View File

@@ -0,0 +1,23 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import "os"
func getLinkInfo(fi os.FileInfo) (uint64, bool) {
return 0, false
}

View File

@@ -0,0 +1,123 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Copyright 2013-2018 Docker, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Original source: https://github.com/moby/moby/blob/v26.0.0/daemon/graphdriver/driver_linux.go
package fs
import (
"path/filepath"
"syscall"
)
// Magic unsigned id of the filesystem in use.
type Magic uint32
const (
// MagicUnsupported is a predefined constant value other than a valid filesystem id.
MagicUnsupported = Magic(0x00000000)
)
const (
// MagicAufs filesystem id for Aufs
MagicAufs = Magic(0x61756673)
// MagicBtrfs filesystem id for Btrfs
MagicBtrfs = Magic(0x9123683E)
// MagicCramfs filesystem id for Cramfs
MagicCramfs = Magic(0x28cd3d45)
// MagicEcryptfs filesystem id for eCryptfs
MagicEcryptfs = Magic(0xf15f)
// MagicExtfs filesystem id for Extfs
MagicExtfs = Magic(0x0000EF53)
// MagicF2fs filesystem id for F2fs
MagicF2fs = Magic(0xF2F52010)
// MagicGPFS filesystem id for GPFS
MagicGPFS = Magic(0x47504653)
// MagicJffs2Fs filesystem if for Jffs2Fs
MagicJffs2Fs = Magic(0x000072b6)
// MagicJfs filesystem id for Jfs
MagicJfs = Magic(0x3153464a)
// MagicNfsFs filesystem id for NfsFs
MagicNfsFs = Magic(0x00006969)
// MagicRAMFs filesystem id for RamFs
MagicRAMFs = Magic(0x858458f6)
// MagicReiserFs filesystem id for ReiserFs
MagicReiserFs = Magic(0x52654973)
// MagicSmbFs filesystem id for SmbFs
MagicSmbFs = Magic(0x0000517B)
// MagicSquashFs filesystem id for SquashFs
MagicSquashFs = Magic(0x73717368)
// MagicTmpFs filesystem id for TmpFs
MagicTmpFs = Magic(0x01021994)
// MagicVxFS filesystem id for VxFs
MagicVxFS = Magic(0xa501fcf5)
// MagicXfs filesystem id for Xfs
MagicXfs = Magic(0x58465342)
// MagicZfs filesystem id for Zfs
MagicZfs = Magic(0x2fc12fc1)
// MagicOverlay filesystem id for overlay
MagicOverlay = Magic(0x794C7630)
)
var (
// FsNames maps filesystem id to name of the filesystem.
FsNames = map[Magic]string{
MagicAufs: "aufs",
MagicBtrfs: "btrfs",
MagicCramfs: "cramfs",
MagicExtfs: "extfs",
MagicF2fs: "f2fs",
MagicGPFS: "gpfs",
MagicJffs2Fs: "jffs2",
MagicJfs: "jfs",
MagicNfsFs: "nfs",
MagicOverlay: "overlayfs",
MagicRAMFs: "ramfs",
MagicReiserFs: "reiserfs",
MagicSmbFs: "smb",
MagicSquashFs: "squashfs",
MagicTmpFs: "tmpfs",
MagicUnsupported: "unsupported",
MagicVxFS: "vxfs",
MagicXfs: "xfs",
MagicZfs: "zfs",
}
)
// GetMagic returns the filesystem id given the path.
func GetMagic(rootpath string) (Magic, error) {
var buf syscall.Statfs_t
if err := syscall.Statfs(filepath.Dir(rootpath), &buf); err != nil {
return 0, err
}
return Magic(buf.Type), nil
}

View File

@@ -0,0 +1,308 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"bytes"
"context"
"errors"
"io"
"os"
"path/filepath"
)
var errTooManyLinks = errors.New("too many links")
type currentPath struct {
path string
f os.FileInfo
fullPath string
}
func pathChange(lower, upper *currentPath) (ChangeKind, string) {
if lower == nil {
if upper == nil {
panic("cannot compare nil paths")
}
return ChangeKindAdd, upper.path
}
if upper == nil {
return ChangeKindDelete, lower.path
}
switch i := directoryCompare(lower.path, upper.path); {
case i < 0:
// File in lower that is not in upper
return ChangeKindDelete, lower.path
case i > 0:
// File in upper that is not in lower
return ChangeKindAdd, upper.path
default:
return ChangeKindModify, upper.path
}
}
func directoryCompare(a, b string) int {
l := len(a)
if len(b) < l {
l = len(b)
}
for i := 0; i < l; i++ {
c1, c2 := a[i], b[i]
if c1 == filepath.Separator {
c1 = byte(0)
}
if c2 == filepath.Separator {
c2 = byte(0)
}
if c1 < c2 {
return -1
}
if c1 > c2 {
return +1
}
}
if len(a) < len(b) {
return -1
}
if len(a) > len(b) {
return +1
}
return 0
}
func sameFile(f1, f2 *currentPath) (bool, error) {
if os.SameFile(f1.f, f2.f) {
return true, nil
}
equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
if err != nil || !equalStat {
return equalStat, err
}
if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
return eq, err
}
// If not a directory also check size, modtime, and content
if !f1.f.IsDir() {
if f1.f.Size() != f2.f.Size() {
return false, nil
}
t1 := f1.f.ModTime()
t2 := f2.f.ModTime()
if t1.Unix() != t2.Unix() {
return false, nil
}
// If the timestamp may have been truncated in both of the
// files, check content of file to determine difference
if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 {
if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink {
return compareSymlinkTarget(f1.fullPath, f2.fullPath)
}
if f1.f.Size() == 0 { // if file sizes are zero length, the files are the same by definition
return true, nil
}
return compareFileContent(f1.fullPath, f2.fullPath)
} else if t1.Nanosecond() != t2.Nanosecond() {
return false, nil
}
}
return true, nil
}
func compareSymlinkTarget(p1, p2 string) (bool, error) {
t1, err := os.Readlink(p1)
if err != nil {
return false, err
}
t2, err := os.Readlink(p2)
if err != nil {
return false, err
}
return t1 == t2, nil
}
const compareChuckSize = 32 * 1024
// compareFileContent compares the content of 2 same sized files
// by comparing each byte.
func compareFileContent(p1, p2 string) (bool, error) {
f1, err := os.Open(p1)
if err != nil {
return false, err
}
defer f1.Close()
f2, err := os.Open(p2)
if err != nil {
return false, err
}
defer f2.Close()
b1 := make([]byte, compareChuckSize)
b2 := make([]byte, compareChuckSize)
for {
n1, err1 := f1.Read(b1)
if err1 != nil && err1 != io.EOF {
return false, err1
}
n2, err2 := f2.Read(b2)
if err2 != nil && err2 != io.EOF {
return false, err2
}
if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
return false, nil
}
if err1 == io.EOF && err2 == io.EOF {
return true, nil
}
}
}
func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error {
return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// Rebase path
path, err = filepath.Rel(root, path)
if err != nil {
return err
}
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == string(os.PathSeparator) {
return nil
}
p := &currentPath{
path: path,
f: f,
fullPath: filepath.Join(root, path),
}
select {
case <-ctx.Done():
return ctx.Err()
case pathC <- p:
return nil
}
})
}
func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
case p := <-pathC:
return p, nil
}
}
// RootPath joins a path with a root, evaluating and bounding any
// symlink to the root directory.
func RootPath(root, path string) (string, error) {
if path == "" {
return root, nil
}
var linksWalked int // to protect against cycles
for {
i := linksWalked
newpath, err := walkLinks(root, path, &linksWalked)
if err != nil {
return "", err
}
path = newpath
if i == linksWalked {
newpath = filepath.Join("/", newpath)
if path == newpath {
return filepath.Join(root, newpath), nil
}
path = newpath
}
}
}
func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) {
if *linksWalked > 255 {
return "", false, errTooManyLinks
}
path = filepath.Join("/", path)
if path == "/" {
return path, false, nil
}
realPath := filepath.Join(root, path)
fi, err := os.Lstat(realPath)
if err != nil {
// If path does not yet exist, treat as non-symlink
if os.IsNotExist(err) {
return path, false, nil
}
return "", false, err
}
if fi.Mode()&os.ModeSymlink == 0 {
return path, false, nil
}
newpath, err = os.Readlink(realPath)
if err != nil {
return "", false, err
}
*linksWalked++
return newpath, true, nil
}
func walkLinks(root, path string, linksWalked *int) (string, error) {
switch dir, file := filepath.Split(path); {
case dir == "":
newpath, _, err := walkLink(root, file, linksWalked)
return newpath, err
case file == "":
if os.IsPathSeparator(dir[len(dir)-1]) {
if dir == "/" {
return dir, nil
}
return walkLinks(root, dir[:len(dir)-1], linksWalked)
}
newpath, _, err := walkLink(root, dir, linksWalked)
return newpath, err
default:
newdir, err := walkLinks(root, dir, linksWalked)
if err != nil {
return "", err
}
newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked)
if err != nil {
return "", err
}
if !islink {
return newpath, nil
}
if filepath.IsAbs(newpath) {
return newpath, nil
}
return filepath.Join(newdir, newpath), nil
}
}

View File

@@ -0,0 +1,70 @@
//go:build darwin || freebsd || netbsd
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"io/fs"
"syscall"
"time"
)
func Atime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Atimespec.Unix()), nil
}
func Ctime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Ctimespec.Unix()), nil
}
func Mtime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Mtimespec.Unix()), nil
}
// StatAtime returns the access time from a stat struct
func StatAtime(st *syscall.Stat_t) syscall.Timespec {
return st.Atimespec
}
// StatCtime returns the created time from a stat struct
func StatCtime(st *syscall.Stat_t) syscall.Timespec {
return st.Ctimespec
}
// StatMtime returns the modified time from a stat struct
func StatMtime(st *syscall.Stat_t) syscall.Timespec {
return st.Mtimespec
}
// StatATimeAsTime returns the access time as a time.Time
func StatATimeAsTime(st *syscall.Stat_t) time.Time {
return time.Unix(st.Atimespec.Unix())
}

View File

@@ -0,0 +1,70 @@
//go:build linux || openbsd || dragonfly || solaris
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"io/fs"
"syscall"
"time"
)
func Atime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Atim.Unix()), nil
}
func Ctime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Ctim.Unix()), nil
}
func Mtime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Stat_t, got %T", st.Sys())
}
return time.Unix(stSys.Mtim.Unix()), nil
}
// StatAtime returns the Atim
func StatAtime(st *syscall.Stat_t) syscall.Timespec {
return st.Atim
}
// StatCtime returns the Ctim
func StatCtime(st *syscall.Stat_t) syscall.Timespec {
return st.Ctim
}
// StatMtime returns the Mtim
func StatMtime(st *syscall.Stat_t) syscall.Timespec {
return st.Mtim
}
// StatATimeAsTime returns st.Atim as a time.Time
func StatATimeAsTime(st *syscall.Stat_t) time.Time {
return time.Unix(st.Atim.Unix())
}

View File

@@ -0,0 +1,33 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"fmt"
"io/fs"
"syscall"
"time"
)
func Atime(st fs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Win32FileAttributeData)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Win32FileAttributeData, got %T", st.Sys())
}
// ref: https://github.com/golang/go/blob/go1.19.2/src/os/types_windows.go#L230
return time.Unix(0, stSys.LastAccessTime.Nanoseconds()), nil
}

View File

@@ -0,0 +1,29 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import "time"
// Gnu tar and the go tar writer don't have sub-second mtime
// precision, which is problematic when we apply changes via tar
// files, we handle this by comparing for exact times, *or* same
// second count and either a or b having exactly 0 nanoseconds
func sameFsTime(a, b time.Time) bool {
return a == b ||
(a.Unix() == b.Unix() &&
(a.Nanosecond() == 0 || b.Nanosecond() == 0))
}

View File

@@ -0,0 +1,32 @@
//go:build !(windows || linux)
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"syscall"
"golang.org/x/sys/unix"
)
func utimesNano(name string, atime, mtime syscall.Timespec) error {
at := unix.NsecToTimespec(atime.Nano())
mt := unix.NsecToTimespec(mtime.Nano())
utimes := [2]unix.Timespec{at, mt}
return unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW)
}

View File

@@ -0,0 +1,162 @@
package confutil
import (
"crypto/rand"
"encoding/hex"
"os"
"path/filepath"
"sync"
"github.com/docker/cli/cli/command"
"github.com/docker/docker/pkg/atomicwriter"
"github.com/moby/buildkit/cmd/buildkitd/config"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
fs "github.com/tonistiigi/fsutil/copy"
)
const defaultBuildKitConfigFile = "buildkitd.default.toml"
type Config struct {
dir string
chowner *chowner
}
type chowner struct {
uid int
gid int
}
type ConfigOption func(*configOptions)
type configOptions struct {
dir string
}
func WithDir(dir string) ConfigOption {
return func(o *configOptions) {
o.dir = dir
}
}
func NewConfig(dockerCli command.Cli, opts ...ConfigOption) *Config {
co := configOptions{}
for _, opt := range opts {
opt(&co)
}
configDir := co.dir
if configDir == "" {
configDir = os.Getenv("BUILDX_CONFIG")
if configDir == "" {
configDir = filepath.Join(filepath.Dir(dockerCli.ConfigFile().Filename), "buildx")
}
}
return &Config{
dir: configDir,
chowner: sudoer(configDir),
}
}
// Dir will look for correct configuration store path;
// if `$BUILDX_CONFIG` is set - use it, otherwise use parent directory
// of Docker config file (i.e. `${DOCKER_CONFIG}/buildx`)
func (c *Config) Dir() string {
return c.dir
}
// BuildKitConfigFile returns the default BuildKit configuration file path
func (c *Config) BuildKitConfigFile() (string, bool) {
f := filepath.Join(c.dir, defaultBuildKitConfigFile)
if _, err := os.Stat(f); err == nil {
return f, true
}
return "", false
}
// MkdirAll creates a directory and all necessary parents within the config dir.
func (c *Config) MkdirAll(dir string, perm os.FileMode) error {
var chown fs.Chowner
if c.chowner != nil {
chown = func(user *fs.User) (*fs.User, error) {
return &fs.User{UID: c.chowner.uid, GID: c.chowner.gid}, nil
}
}
d := filepath.Join(c.dir, dir)
st, err := os.Stat(d)
if err != nil {
if os.IsNotExist(err) {
_, err := fs.MkdirAll(d, perm, chown, nil)
return err
}
return err
}
// if directory already exists, fix the owner if necessary
if c.chowner == nil {
return nil
}
currentOwner := fileOwner(st)
if currentOwner != nil && (currentOwner.uid != c.chowner.uid || currentOwner.gid != c.chowner.gid) {
return os.Chown(d, c.chowner.uid, c.chowner.gid)
}
return nil
}
// AtomicWriteFile writes data to a file within the config dir atomically
func (c *Config) AtomicWriteFile(filename string, data []byte, perm os.FileMode) error {
f := filepath.Join(c.dir, filename)
if err := atomicwriter.WriteFile(f, data, perm); err != nil {
return err
}
if c.chowner == nil {
return nil
}
return os.Chown(f, c.chowner.uid, c.chowner.gid)
}
var nodeIdentifierMu sync.Mutex
func (c *Config) TryNodeIdentifier() (out string) {
nodeIdentifierMu.Lock()
defer nodeIdentifierMu.Unlock()
sessionFilename := ".buildNodeID"
sessionFilepath := filepath.Join(c.Dir(), sessionFilename)
if _, err := os.Lstat(sessionFilepath); err != nil {
if os.IsNotExist(err) { // create a new file with stored randomness
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return out
}
if err := c.AtomicWriteFile(sessionFilename, []byte(hex.EncodeToString(b)), 0600); err != nil {
return out
}
}
}
dt, err := os.ReadFile(sessionFilepath)
if err == nil {
return string(dt)
}
return
}
// LoadConfigTree loads BuildKit config toml tree
func LoadConfigTree(fp string) (*toml.Tree, error) {
f, err := os.Open(fp)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to load config from %s", fp)
}
defer f.Close()
t, err := toml.LoadReader(f)
if err != nil {
return t, errors.Wrap(err, "failed to parse buildkit config")
}
var bkcfg config.Config
if err = t.Unmarshal(&bkcfg); err != nil {
return t, errors.Wrap(err, "failed to parse buildkit config")
}
return t, nil
}

View File

@@ -0,0 +1,60 @@
//go:build !windows
// +build !windows
package confutil
import (
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
)
// sudoer returns the user that invoked the current process with sudo only if
// sudo HOME env matches the home directory of the user that ran sudo and is
// part of configDir.
func sudoer(configDir string) *chowner {
if _, ok := os.LookupEnv("SUDO_COMMAND"); !ok {
return nil
}
suidenv := os.Getenv("SUDO_UID") // https://www.sudo.ws/docs/man/sudo.man/#SUDO_UID
sgidenv := os.Getenv("SUDO_GID") // https://www.sudo.ws/docs/man/sudo.man/#SUDO_GID
if suidenv == "" || sgidenv == "" {
return nil
}
u, err := user.LookupId(suidenv)
if err != nil {
return nil
}
suid, err := strconv.Atoi(suidenv)
if err != nil {
return nil
}
sgid, err := strconv.Atoi(sgidenv)
if err != nil {
return nil
}
home, _ := os.UserHomeDir()
if home == "" || u.HomeDir != home {
return nil
}
if ok, _ := isSubPath(home, configDir); !ok {
return nil
}
return &chowner{uid: suid, gid: sgid}
}
func fileOwner(fi os.FileInfo) *chowner {
st := fi.Sys().(*syscall.Stat_t)
return &chowner{uid: int(st.Uid), gid: int(st.Gid)}
}
func isSubPath(basePath, subPath string) (bool, error) {
rel, err := filepath.Rel(basePath, subPath)
if err != nil {
return false, err
}
return !strings.HasPrefix(rel, "..") && rel != ".", nil
}

View File

@@ -0,0 +1,11 @@
package confutil
import "os"
func sudoer(_ string) *chowner {
return nil
}
func fileOwner(_ os.FileInfo) *chowner {
return nil
}

View File

@@ -0,0 +1,135 @@
package confutil
import (
"bytes"
"io"
"os"
"path"
"regexp"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
)
const (
// DefaultBuildKitStateDir and DefaultBuildKitConfigDir are the location
// where buildkitd inside the container stores its state. Some drivers
// create a Linux container, so this should match the location for Linux,
// as defined in: https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
DefaultBuildKitStateDir = "/var/lib/buildkit"
DefaultBuildKitConfigDir = "/etc/buildkit"
)
var reInvalidCertsDir = regexp.MustCompile(`[^a-zA-Z0-9.-]+`)
// LoadConfigFiles creates a temp directory with BuildKit config and
// registry certificates ready to be copied to a container.
func LoadConfigFiles(bkconfig string) (map[string][]byte, error) {
if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
} else if err != nil {
return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
}
// Load config tree
btoml, err := LoadConfigTree(bkconfig)
if err != nil {
return nil, err
}
m := make(map[string][]byte)
// Iterate through registry config to copy certs and update
// BuildKit config with the underlying certs' path in the container.
//
// The following BuildKit config:
//
// [registry."myregistry.io"]
// ca=["/etc/config/myca.pem"]
// [[registry."myregistry.io".keypair]]
// key="/etc/config/key.pem"
// cert="/etc/config/cert.pem"
//
// will be translated in the container as:
//
// [registry."myregistry.io"]
// ca=["/etc/buildkit/certs/myregistry.io/myca.pem"]
// [[registry."myregistry.io".keypair]]
// key="/etc/buildkit/certs/myregistry.io/key.pem"
// cert="/etc/buildkit/certs/myregistry.io/cert.pem"
if btoml.Has("registry") {
for regName := range btoml.GetArray("registry").(*toml.Tree).Values() {
regConf := btoml.GetPath([]string{"registry", regName}).(*toml.Tree)
if regConf == nil {
continue
}
pfx := path.Join("certs", reInvalidCertsDir.ReplaceAllString(regName, "_"))
if regConf.Has("ca") {
regCAs := regConf.GetArray("ca").([]string)
if len(regCAs) > 0 {
var cas []string
for _, ca := range regCAs {
fp := path.Join(pfx, path.Base(ca))
cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(ca)
if err != nil {
return nil, errors.Wrapf(err, "failed to read CA file: %s", ca)
}
m[fp] = dt
}
regConf.Set("ca", cas)
}
}
if regConf.Has("keypair") {
regKeyPairs := regConf.GetArray("keypair").([]*toml.Tree)
if len(regKeyPairs) == 0 {
continue
}
for _, kp := range regKeyPairs {
if kp == nil {
continue
}
key := kp.Get("key").(string)
if len(key) > 0 {
fp := path.Join(pfx, path.Base(key))
kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(key)
if err != nil {
return nil, errors.Wrapf(err, "failed to read key file: %s", key)
}
m[fp] = dt
}
cert := kp.Get("cert").(string)
if len(cert) > 0 {
fp := path.Join(pfx, path.Base(cert))
kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(cert)
if err != nil {
return nil, errors.Wrapf(err, "failed to read cert file: %s", cert)
}
m[fp] = dt
}
}
}
}
}
b := bytes.NewBuffer(nil)
_, err = btoml.WriteTo(b)
if err != nil {
return nil, err
}
m["buildkitd.toml"] = b.Bytes()
return m, nil
}
func readFile(fp string) ([]byte, error) {
sf, err := os.Open(fp)
if err != nil {
return nil, err
}
defer sf.Close()
return io.ReadAll(io.LimitReader(sf, 1024*1024))
}

View File

@@ -0,0 +1,15 @@
package confutil
import (
"os"
"strconv"
)
// IsExperimental checks if the experimental flag has been configured.
func IsExperimental() bool {
if v, ok := os.LookupEnv("BUILDX_EXPERIMENTAL"); ok {
vv, _ := strconv.ParseBool(v)
return vv
}
return false
}

View File

@@ -0,0 +1,50 @@
package confutil
import (
"os"
"strconv"
)
// MetadataProvenanceMode is the type for setting provenance in the metadata
// file
type MetadataProvenanceMode string
const (
// MetadataProvenanceModeMin sets minimal provenance (default)
MetadataProvenanceModeMin MetadataProvenanceMode = "min"
// MetadataProvenanceModeMax sets full provenance
MetadataProvenanceModeMax MetadataProvenanceMode = "max"
// MetadataProvenanceModeDisabled doesn't set provenance
MetadataProvenanceModeDisabled MetadataProvenanceMode = "disabled"
)
// MetadataProvenance returns the metadata provenance mode from
// BUILDX_METADATA_PROVENANCE environment variable
func MetadataProvenance() MetadataProvenanceMode {
return ParseMetadataProvenance(os.Getenv("BUILDX_METADATA_PROVENANCE"))
}
// ParseMetadataProvenance parses the metadata provenance mode from a string
func ParseMetadataProvenance(inp string) MetadataProvenanceMode {
switch inp {
case "min":
return MetadataProvenanceModeMin
case "max":
return MetadataProvenanceModeMax
case "disabled":
return MetadataProvenanceModeDisabled
}
if ok, err := strconv.ParseBool(inp); err == nil && !ok {
return MetadataProvenanceModeDisabled
}
return MetadataProvenanceModeMin
}
// MetadataWarningsEnabled returns whether metadata warnings are enabled from
// BUILDX_METADATA_WARNINGS environment variable (default false)
func MetadataWarningsEnabled() bool {
if ok, err := strconv.ParseBool(os.Getenv("BUILDX_METADATA_WARNINGS")); err == nil {
return ok
}
return false
}

View File

@@ -0,0 +1,577 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package command
import (
"context"
"fmt"
"io"
"os"
"runtime"
"strconv"
"sync"
"time"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/configfile"
dcontext "github.com/docker/cli/cli/context"
"github.com/docker/cli/cli/context/docker"
"github.com/docker/cli/cli/context/store"
"github.com/docker/cli/cli/debug"
cliflags "github.com/docker/cli/cli/flags"
"github.com/docker/cli/cli/streams"
"github.com/docker/cli/cli/version"
dopts "github.com/docker/cli/opts"
"github.com/docker/docker/api"
"github.com/docker/docker/api/types/build"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
const defaultInitTimeout = 2 * time.Second
// Streams is an interface which exposes the standard input and output streams
type Streams interface {
In() *streams.In
Out() *streams.Out
Err() *streams.Out
}
// Cli represents the docker command line client.
type Cli interface {
Client() client.APIClient
Streams
SetIn(in *streams.In)
Apply(ops ...CLIOption) error
config.Provider
ServerInfo() ServerInfo
DefaultVersion() string
CurrentVersion() string
ContentTrustEnabled() bool
BuildKitEnabled() (bool, error)
ContextStore() store.Store
CurrentContext() string
DockerEndpoint() docker.Endpoint
TelemetryClient
}
// DockerCli is an instance the docker command line client.
// Instances of the client should be created using the [NewDockerCli]
// constructor to make sure they are properly initialized with defaults
// set.
type DockerCli struct {
configFile *configfile.ConfigFile
options *cliflags.ClientOptions
in *streams.In
out *streams.Out
err *streams.Out
client client.APIClient
serverInfo ServerInfo
contentTrust bool
contextStore store.Store
currentContext string
init sync.Once
initErr error
dockerEndpoint docker.Endpoint
contextStoreConfig *store.Config
initTimeout time.Duration
res telemetryResource
// baseCtx is the base context used for internal operations. In the future
// this may be replaced by explicitly passing a context to functions that
// need it.
baseCtx context.Context
enableGlobalMeter, enableGlobalTracer bool
}
// DefaultVersion returns [api.DefaultVersion].
func (*DockerCli) DefaultVersion() string {
return api.DefaultVersion
}
// CurrentVersion returns the API version currently negotiated, or the default
// version otherwise.
func (cli *DockerCli) CurrentVersion() string {
_ = cli.initialize()
if cli.client == nil {
return api.DefaultVersion
}
return cli.client.ClientVersion()
}
// Client returns the APIClient
func (cli *DockerCli) Client() client.APIClient {
if err := cli.initialize(); err != nil {
_, _ = fmt.Fprintln(cli.Err(), "Failed to initialize:", err)
os.Exit(1)
}
return cli.client
}
// Out returns the writer used for stdout
func (cli *DockerCli) Out() *streams.Out {
return cli.out
}
// Err returns the writer used for stderr
func (cli *DockerCli) Err() *streams.Out {
return cli.err
}
// SetIn sets the reader used for stdin
func (cli *DockerCli) SetIn(in *streams.In) {
cli.in = in
}
// In returns the reader used for stdin
func (cli *DockerCli) In() *streams.In {
return cli.in
}
// ShowHelp shows the command help.
func ShowHelp(err io.Writer) func(*cobra.Command, []string) error {
return func(cmd *cobra.Command, args []string) error {
cmd.SetOut(err)
cmd.HelpFunc()(cmd, args)
return nil
}
}
// ConfigFile returns the ConfigFile
func (cli *DockerCli) ConfigFile() *configfile.ConfigFile {
// TODO(thaJeztah): when would this happen? Is this only in tests (where cli.Initialize() is not called first?)
if cli.configFile == nil {
cli.configFile = config.LoadDefaultConfigFile(cli.err)
}
return cli.configFile
}
// ServerInfo returns the server version details for the host this client is
// connected to
func (cli *DockerCli) ServerInfo() ServerInfo {
_ = cli.initialize()
return cli.serverInfo
}
// ContentTrustEnabled returns whether content trust has been enabled by an
// environment variable.
func (cli *DockerCli) ContentTrustEnabled() bool {
return cli.contentTrust
}
// BuildKitEnabled returns buildkit is enabled or not.
func (cli *DockerCli) BuildKitEnabled() (bool, error) {
// use DOCKER_BUILDKIT env var value if set and not empty
if v := os.Getenv("DOCKER_BUILDKIT"); v != "" {
enabled, err := strconv.ParseBool(v)
if err != nil {
return false, errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value")
}
return enabled, nil
}
// if a builder alias is defined, we are using BuildKit
aliasMap := cli.ConfigFile().Aliases
if _, ok := aliasMap["builder"]; ok {
return true, nil
}
si := cli.ServerInfo()
if si.BuildkitVersion == build.BuilderBuildKit {
// The daemon advertised BuildKit as the preferred builder; this may
// be either a Linux daemon or a Windows daemon with experimental
// BuildKit support enabled.
return true, nil
}
// otherwise, assume BuildKit is enabled for Linux, but disabled for
// Windows / WCOW, which does not yet support BuildKit by default.
return si.OSType != "windows", nil
}
// HooksEnabled returns whether plugin hooks are enabled.
func (cli *DockerCli) HooksEnabled() bool {
// use DOCKER_CLI_HOOKS env var value if set and not empty
if v := os.Getenv("DOCKER_CLI_HOOKS"); v != "" {
enabled, err := strconv.ParseBool(v)
if err != nil {
return false
}
return enabled
}
// legacy support DOCKER_CLI_HINTS env var
if v := os.Getenv("DOCKER_CLI_HINTS"); v != "" {
enabled, err := strconv.ParseBool(v)
if err != nil {
return false
}
return enabled
}
featuresMap := cli.ConfigFile().Features
if v, ok := featuresMap["hooks"]; ok {
enabled, err := strconv.ParseBool(v)
if err != nil {
return false
}
return enabled
}
// default to false
return false
}
// Initialize the dockerCli runs initialization that must happen after command
// line flags are parsed.
func (cli *DockerCli) Initialize(opts *cliflags.ClientOptions, ops ...CLIOption) error {
for _, o := range ops {
if err := o(cli); err != nil {
return err
}
}
cliflags.SetLogLevel(opts.LogLevel)
if opts.ConfigDir != "" {
config.SetDir(opts.ConfigDir)
}
if opts.Debug {
debug.Enable()
}
if opts.Context != "" && len(opts.Hosts) > 0 {
return errors.New("conflicting options: cannot specify both --host and --context")
}
if cli.contextStoreConfig == nil {
// This path can be hit when calling Initialize on a DockerCli that's
// not constructed through [NewDockerCli]. Using the default context
// store without a config set will result in Endpoints from contexts
// not being type-mapped correctly, and used as a generic "map[string]any",
// instead of a [docker.EndpointMeta].
//
// When looking up the API endpoint (using [EndpointFromContext]), no
// endpoint will be found, and a default, empty endpoint will be used
// instead which in its turn, causes newAPIClientFromEndpoint to
// be initialized with the default config instead of settings for
// the current context (which may mean; connecting with the wrong
// endpoint and/or TLS Config to be missing).
//
// [EndpointFromContext]: https://github.com/docker/cli/blob/33494921b80fd0b5a06acc3a34fa288de4bb2e6b/cli/context/docker/load.go#L139-L149
if err := WithDefaultContextStoreConfig()(cli); err != nil {
return err
}
}
cli.options = opts
cli.configFile = config.LoadDefaultConfigFile(cli.err)
cli.currentContext = resolveContextName(cli.options, cli.configFile)
cli.contextStore = &ContextStoreWithDefault{
Store: store.New(config.ContextStoreDir(), *cli.contextStoreConfig),
Resolver: func() (*DefaultContext, error) {
return ResolveDefaultContext(cli.options, *cli.contextStoreConfig)
},
}
// TODO(krissetto): pass ctx to the funcs instead of using this
if cli.enableGlobalMeter {
cli.createGlobalMeterProvider(cli.baseCtx)
}
if cli.enableGlobalTracer {
cli.createGlobalTracerProvider(cli.baseCtx)
}
filterResourceAttributesEnvvar()
return nil
}
// NewAPIClientFromFlags creates a new APIClient from command line flags
func NewAPIClientFromFlags(opts *cliflags.ClientOptions, configFile *configfile.ConfigFile) (client.APIClient, error) {
if opts.Context != "" && len(opts.Hosts) > 0 {
return nil, errors.New("conflicting options: cannot specify both --host and --context")
}
storeConfig := DefaultContextStoreConfig()
contextStore := &ContextStoreWithDefault{
Store: store.New(config.ContextStoreDir(), storeConfig),
Resolver: func() (*DefaultContext, error) {
return ResolveDefaultContext(opts, storeConfig)
},
}
endpoint, err := resolveDockerEndpoint(contextStore, resolveContextName(opts, configFile))
if err != nil {
return nil, errors.Wrap(err, "unable to resolve docker endpoint")
}
return newAPIClientFromEndpoint(endpoint, configFile)
}
func newAPIClientFromEndpoint(ep docker.Endpoint, configFile *configfile.ConfigFile) (client.APIClient, error) {
opts, err := ep.ClientOpts()
if err != nil {
return nil, err
}
if len(configFile.HTTPHeaders) > 0 {
opts = append(opts, client.WithHTTPHeaders(configFile.HTTPHeaders))
}
opts = append(opts, withCustomHeadersFromEnv(), client.WithUserAgent(UserAgent()))
return client.NewClientWithOpts(opts...)
}
func resolveDockerEndpoint(s store.Reader, contextName string) (docker.Endpoint, error) {
if s == nil {
return docker.Endpoint{}, errors.New("no context store initialized")
}
ctxMeta, err := s.GetMetadata(contextName)
if err != nil {
return docker.Endpoint{}, err
}
epMeta, err := docker.EndpointFromContext(ctxMeta)
if err != nil {
return docker.Endpoint{}, err
}
return docker.WithTLSData(s, contextName, epMeta)
}
// Resolve the Docker endpoint for the default context (based on config, env vars and CLI flags)
func resolveDefaultDockerEndpoint(opts *cliflags.ClientOptions) (docker.Endpoint, error) {
// defaultToTLS determines whether we should use a TLS host as default
// if nothing was configured by the user.
defaultToTLS := opts.TLSOptions != nil
host, err := getServerHost(opts.Hosts, defaultToTLS)
if err != nil {
return docker.Endpoint{}, err
}
var (
skipTLSVerify bool
tlsData *dcontext.TLSData
)
if opts.TLSOptions != nil {
skipTLSVerify = opts.TLSOptions.InsecureSkipVerify
tlsData, err = dcontext.TLSDataFromFiles(opts.TLSOptions.CAFile, opts.TLSOptions.CertFile, opts.TLSOptions.KeyFile)
if err != nil {
return docker.Endpoint{}, err
}
}
return docker.Endpoint{
EndpointMeta: docker.EndpointMeta{
Host: host,
SkipTLSVerify: skipTLSVerify,
},
TLSData: tlsData,
}, nil
}
func (cli *DockerCli) getInitTimeout() time.Duration {
if cli.initTimeout != 0 {
return cli.initTimeout
}
return defaultInitTimeout
}
func (cli *DockerCli) initializeFromClient() {
ctx, cancel := context.WithTimeout(cli.baseCtx, cli.getInitTimeout())
defer cancel()
ping, err := cli.client.Ping(ctx)
if err != nil {
// Default to true if we fail to connect to daemon
cli.serverInfo = ServerInfo{HasExperimental: true}
if ping.APIVersion != "" {
cli.client.NegotiateAPIVersionPing(ping)
}
return
}
cli.serverInfo = ServerInfo{
HasExperimental: ping.Experimental,
OSType: ping.OSType,
BuildkitVersion: ping.BuilderVersion,
SwarmStatus: ping.SwarmStatus,
}
cli.client.NegotiateAPIVersionPing(ping)
}
// ContextStore returns the ContextStore
func (cli *DockerCli) ContextStore() store.Store {
return cli.contextStore
}
// CurrentContext returns the current context name, based on flags,
// environment variables and the cli configuration file, in the following
// order of preference:
//
// 1. The "--context" command-line option.
// 2. The "DOCKER_CONTEXT" environment variable ([EnvOverrideContext]).
// 3. The current context as configured through the in "currentContext"
// field in the CLI configuration file ("~/.docker/config.json").
// 4. If no context is configured, use the "default" context.
//
// # Fallbacks for backward-compatibility
//
// To preserve backward-compatibility with the "pre-contexts" behavior,
// the "default" context is used if:
//
// - The "--host" option is set
// - The "DOCKER_HOST" ([client.EnvOverrideHost]) environment variable is set
// to a non-empty value.
//
// In these cases, the default context is used, which uses the host as
// specified in "DOCKER_HOST", and TLS config from flags/env vars.
//
// Setting both the "--context" and "--host" flags is ambiguous and results
// in an error when the cli is started.
//
// CurrentContext does not validate if the given context exists or if it's
// valid; errors may occur when trying to use it.
func (cli *DockerCli) CurrentContext() string {
return cli.currentContext
}
// CurrentContext returns the current context name, based on flags,
// environment variables and the cli configuration file. It does not
// validate if the given context exists or if it's valid; errors may
// occur when trying to use it.
//
// Refer to [DockerCli.CurrentContext] above for further details.
func resolveContextName(opts *cliflags.ClientOptions, cfg *configfile.ConfigFile) string {
if opts != nil && opts.Context != "" {
return opts.Context
}
if opts != nil && len(opts.Hosts) > 0 {
return DefaultContextName
}
if os.Getenv(client.EnvOverrideHost) != "" {
return DefaultContextName
}
if ctxName := os.Getenv(EnvOverrideContext); ctxName != "" {
return ctxName
}
if cfg != nil && cfg.CurrentContext != "" {
// We don't validate if this context exists: errors may occur when trying to use it.
return cfg.CurrentContext
}
return DefaultContextName
}
// DockerEndpoint returns the current docker endpoint
func (cli *DockerCli) DockerEndpoint() docker.Endpoint {
if err := cli.initialize(); err != nil {
// Note that we're not terminating here, as this function may be used
// in cases where we're able to continue.
_, _ = fmt.Fprintln(cli.Err(), cli.initErr)
}
return cli.dockerEndpoint
}
func (cli *DockerCli) getDockerEndPoint() (ep docker.Endpoint, err error) {
cn := cli.CurrentContext()
if cn == DefaultContextName {
return resolveDefaultDockerEndpoint(cli.options)
}
return resolveDockerEndpoint(cli.contextStore, cn)
}
func (cli *DockerCli) initialize() error {
cli.init.Do(func() {
cli.dockerEndpoint, cli.initErr = cli.getDockerEndPoint()
if cli.initErr != nil {
cli.initErr = errors.Wrap(cli.initErr, "unable to resolve docker endpoint")
return
}
if cli.client == nil {
if cli.client, cli.initErr = newAPIClientFromEndpoint(cli.dockerEndpoint, cli.configFile); cli.initErr != nil {
return
}
}
if cli.baseCtx == nil {
cli.baseCtx = context.Background()
}
cli.initializeFromClient()
})
return cli.initErr
}
// Apply all the operation on the cli
func (cli *DockerCli) Apply(ops ...CLIOption) error {
for _, op := range ops {
if err := op(cli); err != nil {
return err
}
}
return nil
}
// ServerInfo stores details about the supported features and platform of the
// server
type ServerInfo struct {
HasExperimental bool
OSType string
BuildkitVersion build.BuilderVersion
// SwarmStatus provides information about the current swarm status of the
// engine, obtained from the "Swarm" header in the API response.
//
// It can be a nil struct if the API version does not provide this header
// in the ping response, or if an error occurred, in which case the client
// should use other ways to get the current swarm status, such as the /swarm
// endpoint.
SwarmStatus *swarm.Status
}
// NewDockerCli returns a DockerCli instance with all operators applied on it.
// It applies by default the standard streams, and the content trust from
// environment.
func NewDockerCli(ops ...CLIOption) (*DockerCli, error) {
defaultOps := []CLIOption{
WithContentTrustFromEnv(),
WithDefaultContextStoreConfig(),
WithStandardStreams(),
}
ops = append(defaultOps, ops...)
cli := &DockerCli{baseCtx: context.Background()}
if err := cli.Apply(ops...); err != nil {
return nil, err
}
return cli, nil
}
func getServerHost(hosts []string, defaultToTLS bool) (string, error) {
switch len(hosts) {
case 0:
return dopts.ParseHost(defaultToTLS, os.Getenv(client.EnvOverrideHost))
case 1:
return dopts.ParseHost(defaultToTLS, hosts[0])
default:
return "", errors.New("Specify only one -H")
}
}
// UserAgent returns the user agent string used for making API requests
func UserAgent() string {
return "Docker-Client/" + version.Version + " (" + runtime.GOOS + ")"
}
var defaultStoreEndpoints = []store.NamedTypeGetter{
store.EndpointTypeGetter(docker.DockerEndpoint, func() any { return &docker.EndpointMeta{} }),
}
// RegisterDefaultStoreEndpoints registers a new named endpoint
// metadata type with the default context store config, so that
// endpoint will be supported by stores using the config returned by
// DefaultContextStoreConfig.
func RegisterDefaultStoreEndpoints(ep ...store.NamedTypeGetter) {
defaultStoreEndpoints = append(defaultStoreEndpoints, ep...)
}
// DefaultContextStoreConfig returns a new store.Config with the default set of endpoints configured.
func DefaultContextStoreConfig() store.Config {
return store.NewConfig(
func() any { return &DockerContext{} },
defaultStoreEndpoints...,
)
}

View File

@@ -0,0 +1,240 @@
package command
import (
"context"
"encoding/csv"
"io"
"net/http"
"os"
"strconv"
"strings"
"github.com/docker/cli/cli/streams"
"github.com/docker/docker/client"
"github.com/moby/term"
"github.com/pkg/errors"
)
// CLIOption is a functional argument to apply options to a [DockerCli]. These
// options can be passed to [NewDockerCli] to initialize a new CLI, or
// applied with [DockerCli.Initialize] or [DockerCli.Apply].
type CLIOption func(cli *DockerCli) error
// WithStandardStreams sets a cli in, out and err streams with the standard streams.
func WithStandardStreams() CLIOption {
return func(cli *DockerCli) error {
// Set terminal emulation based on platform as required.
stdin, stdout, stderr := term.StdStreams()
cli.in = streams.NewIn(stdin)
cli.out = streams.NewOut(stdout)
cli.err = streams.NewOut(stderr)
return nil
}
}
// WithBaseContext sets the base context of a cli. It is used to propagate
// the context from the command line to the client.
func WithBaseContext(ctx context.Context) CLIOption {
return func(cli *DockerCli) error {
cli.baseCtx = ctx
return nil
}
}
// WithCombinedStreams uses the same stream for the output and error streams.
func WithCombinedStreams(combined io.Writer) CLIOption {
return func(cli *DockerCli) error {
s := streams.NewOut(combined)
cli.out = s
cli.err = s
return nil
}
}
// WithInputStream sets a cli input stream.
func WithInputStream(in io.ReadCloser) CLIOption {
return func(cli *DockerCli) error {
cli.in = streams.NewIn(in)
return nil
}
}
// WithOutputStream sets a cli output stream.
func WithOutputStream(out io.Writer) CLIOption {
return func(cli *DockerCli) error {
cli.out = streams.NewOut(out)
return nil
}
}
// WithErrorStream sets a cli error stream.
func WithErrorStream(err io.Writer) CLIOption {
return func(cli *DockerCli) error {
cli.err = streams.NewOut(err)
return nil
}
}
// WithContentTrustFromEnv enables content trust on a cli from environment variable DOCKER_CONTENT_TRUST value.
func WithContentTrustFromEnv() CLIOption {
return func(cli *DockerCli) error {
cli.contentTrust = false
if e := os.Getenv("DOCKER_CONTENT_TRUST"); e != "" {
if t, err := strconv.ParseBool(e); t || err != nil {
// treat any other value as true
cli.contentTrust = true
}
}
return nil
}
}
// WithContentTrust enables content trust on a cli.
func WithContentTrust(enabled bool) CLIOption {
return func(cli *DockerCli) error {
cli.contentTrust = enabled
return nil
}
}
// WithDefaultContextStoreConfig configures the cli to use the default context store configuration.
func WithDefaultContextStoreConfig() CLIOption {
return func(cli *DockerCli) error {
cfg := DefaultContextStoreConfig()
cli.contextStoreConfig = &cfg
return nil
}
}
// WithAPIClient configures the cli to use the given API client.
func WithAPIClient(c client.APIClient) CLIOption {
return func(cli *DockerCli) error {
cli.client = c
return nil
}
}
// WithInitializeClient is passed to [DockerCli.Initialize] to initialize
// an API Client for use by the CLI.
func WithInitializeClient(makeClient func(*DockerCli) (client.APIClient, error)) CLIOption {
return func(cli *DockerCli) error {
c, err := makeClient(cli)
if err != nil {
return err
}
return WithAPIClient(c)(cli)
}
}
// envOverrideHTTPHeaders is the name of the environment-variable that can be
// used to set custom HTTP headers to be sent by the client. This environment
// variable is the equivalent to the HttpHeaders field in the configuration
// file.
//
// WARNING: If both config and environment-variable are set, the environment
// variable currently overrides all headers set in the configuration file.
// This behavior may change in a future update, as we are considering the
// environment variable to be appending to existing headers (and to only
// override headers with the same name).
//
// While this env-var allows for custom headers to be set, it does not allow
// for built-in headers (such as "User-Agent", if set) to be overridden.
// Also see [client.WithHTTPHeaders] and [client.WithUserAgent].
//
// This environment variable can be used in situations where headers must be
// set for a specific invocation of the CLI, but should not be set by default,
// and therefore cannot be set in the config-file.
//
// envOverrideHTTPHeaders accepts a comma-separated (CSV) list of key=value pairs,
// where key must be a non-empty, valid MIME header format. Whitespaces surrounding
// the key are trimmed, and the key is normalised. Whitespaces in values are
// preserved, but "key=value" pairs with an empty value (e.g. "key=") are ignored.
// Tuples without a "=" produce an error.
//
// It follows CSV rules for escaping, allowing "key=value" pairs to be quoted
// if they must contain commas, which allows for multiple values for a single
// header to be set. If a key is repeated in the list, later values override
// prior values.
//
// For example, the following value:
//
// one=one-value,"two=two,value","three= a value with whitespace ",four=,five=five=one,five=five-two
//
// Produces four headers (four is omitted as it has an empty value set):
//
// - one (value is "one-value")
// - two (value is "two,value")
// - three (value is " a value with whitespace ")
// - five (value is "five-two", the later value has overridden the prior value)
const envOverrideHTTPHeaders = "DOCKER_CUSTOM_HEADERS"
// withCustomHeadersFromEnv overriding custom HTTP headers to be sent by the
// client through the [envOverrideHTTPHeaders] environment-variable. This
// environment variable is the equivalent to the HttpHeaders field in the
// configuration file.
//
// WARNING: If both config and environment-variable are set, the environment-
// variable currently overrides all headers set in the configuration file.
// This behavior may change in a future update, as we are considering the
// environment-variable to be appending to existing headers (and to only
// override headers with the same name).
//
// TODO(thaJeztah): this is a client Option, and should be moved to the client. It is non-exported for that reason.
func withCustomHeadersFromEnv() client.Opt {
return func(apiClient *client.Client) error {
value := os.Getenv(envOverrideHTTPHeaders)
if value == "" {
return nil
}
csvReader := csv.NewReader(strings.NewReader(value))
fields, err := csvReader.Read()
if err != nil {
return invalidParameter(errors.Errorf(
"failed to parse custom headers from %s environment variable: value must be formatted as comma-separated key=value pairs",
envOverrideHTTPHeaders,
))
}
if len(fields) == 0 {
return nil
}
env := map[string]string{}
for _, kv := range fields {
k, v, hasValue := strings.Cut(kv, "=")
// Only strip whitespace in keys; preserve whitespace in values.
k = strings.TrimSpace(k)
if k == "" {
return invalidParameter(errors.Errorf(
`failed to set custom headers from %s environment variable: value contains a key=value pair with an empty key: '%s'`,
envOverrideHTTPHeaders, kv,
))
}
// We don't currently allow empty key=value pairs, and produce an error.
// This is something we could allow in future (e.g. to read value
// from an environment variable with the same name). In the meantime,
// produce an error to prevent users from depending on this.
if !hasValue {
return invalidParameter(errors.Errorf(
`failed to set custom headers from %s environment variable: missing "=" in key=value pair: '%s'`,
envOverrideHTTPHeaders, kv,
))
}
env[http.CanonicalHeaderKey(k)] = v
}
if len(env) == 0 {
// We should probably not hit this case, as we don't skip values
// (only return errors), but we don't want to discard existing
// headers with an empty set.
return nil
}
// TODO(thaJeztah): add a client.WithExtraHTTPHeaders() function to allow these headers to be _added_ to existing ones, instead of _replacing_
// see https://github.com/docker/cli/pull/5098#issuecomment-2147403871 (when updating, also update the WARNING in the function and env-var GoDoc)
return client.WithHTTPHeaders(env)(apiClient)
}
}

View File

@@ -0,0 +1,68 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package command
import (
"encoding/json"
"errors"
"github.com/docker/cli/cli/context/store"
)
// DockerContext is a typed representation of what we put in Context metadata
type DockerContext struct {
Description string
AdditionalFields map[string]any
}
// MarshalJSON implements custom JSON marshalling
func (dc DockerContext) MarshalJSON() ([]byte, error) {
s := map[string]any{}
if dc.Description != "" {
s["Description"] = dc.Description
}
if dc.AdditionalFields != nil {
for k, v := range dc.AdditionalFields {
s[k] = v
}
}
return json.Marshal(s)
}
// UnmarshalJSON implements custom JSON marshalling
func (dc *DockerContext) UnmarshalJSON(payload []byte) error {
var data map[string]any
if err := json.Unmarshal(payload, &data); err != nil {
return err
}
for k, v := range data {
switch k {
case "Description":
dc.Description = v.(string)
default:
if dc.AdditionalFields == nil {
dc.AdditionalFields = make(map[string]any)
}
dc.AdditionalFields[k] = v
}
}
return nil
}
// GetDockerContext extracts metadata from stored context metadata
func GetDockerContext(storeMetadata store.Metadata) (DockerContext, error) {
if storeMetadata.Metadata == nil {
// can happen if we save endpoints before assigning a context metadata
// it is totally valid, and we should return a default initialized value
return DockerContext{}, nil
}
res, ok := storeMetadata.Metadata.(DockerContext)
if !ok {
return DockerContext{}, errors.New("context metadata is not a valid DockerContext")
}
if storeMetadata.Name == DefaultContextName {
res.Description = "Current DOCKER_HOST based configuration"
}
return res, nil
}

View File

@@ -0,0 +1,201 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package command
import (
"github.com/docker/cli/cli/context/docker"
"github.com/docker/cli/cli/context/store"
cliflags "github.com/docker/cli/cli/flags"
"github.com/pkg/errors"
)
const (
// DefaultContextName is the name reserved for the default context (config & env based)
DefaultContextName = "default"
// EnvOverrideContext is the name of the environment variable that can be
// used to override the context to use. If set, it overrides the context
// that's set in the CLI's configuration file, but takes no effect if the
// "DOCKER_HOST" env-var is set (which takes precedence.
EnvOverrideContext = "DOCKER_CONTEXT"
)
// DefaultContext contains the default context data for all endpoints
type DefaultContext struct {
Meta store.Metadata
TLS store.ContextTLSData
}
// DefaultContextResolver is a function which resolves the default context base on the configuration and the env variables
type DefaultContextResolver func() (*DefaultContext, error)
// ContextStoreWithDefault implements the store.Store interface with a support for the default context
type ContextStoreWithDefault struct {
store.Store
Resolver DefaultContextResolver
}
// EndpointDefaultResolver is implemented by any EndpointMeta object
// which wants to be able to populate the store with whatever their default is.
type EndpointDefaultResolver interface {
// ResolveDefault returns values suitable for storing in store.Metadata.Endpoints
// and store.ContextTLSData.Endpoints.
//
// An error is only returned for something fatal, not simply
// the lack of a default (e.g. because the config file which
// would contain it is missing). If there is no default then
// returns nil, nil, nil.
//
//nolint:dupword // ignore "Duplicate words (nil,) found"
ResolveDefault() (any, *store.EndpointTLSData, error)
}
// ResolveDefaultContext creates a Metadata for the current CLI invocation parameters
func ResolveDefaultContext(opts *cliflags.ClientOptions, config store.Config) (*DefaultContext, error) {
contextTLSData := store.ContextTLSData{
Endpoints: make(map[string]store.EndpointTLSData),
}
contextMetadata := store.Metadata{
Endpoints: make(map[string]any),
Metadata: DockerContext{
Description: "",
},
Name: DefaultContextName,
}
dockerEP, err := resolveDefaultDockerEndpoint(opts)
if err != nil {
return nil, err
}
contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP.EndpointMeta
if dockerEP.TLSData != nil {
contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerEP.TLSData.ToStoreTLSData()
}
if err := config.ForeachEndpointType(func(n string, get store.TypeGetter) error {
if n == docker.DockerEndpoint { // handled above
return nil
}
ep := get()
if i, ok := ep.(EndpointDefaultResolver); ok {
meta, tls, err := i.ResolveDefault()
if err != nil {
return err
}
if meta == nil {
return nil
}
contextMetadata.Endpoints[n] = meta
if tls != nil {
contextTLSData.Endpoints[n] = *tls
}
}
// Nothing to be done
return nil
}); err != nil {
return nil, err
}
return &DefaultContext{Meta: contextMetadata, TLS: contextTLSData}, nil
}
// List implements store.Store's List
func (s *ContextStoreWithDefault) List() ([]store.Metadata, error) {
contextList, err := s.Store.List()
if err != nil {
return nil, err
}
defaultContext, err := s.Resolver()
if err != nil {
return nil, err
}
return append(contextList, defaultContext.Meta), nil
}
// CreateOrUpdate is not allowed for the default context and fails
func (s *ContextStoreWithDefault) CreateOrUpdate(meta store.Metadata) error {
if meta.Name == DefaultContextName {
return invalidParameter(errors.New("default context cannot be created nor updated"))
}
return s.Store.CreateOrUpdate(meta)
}
// Remove is not allowed for the default context and fails
func (s *ContextStoreWithDefault) Remove(name string) error {
if name == DefaultContextName {
return invalidParameter(errors.New("default context cannot be removed"))
}
return s.Store.Remove(name)
}
// GetMetadata implements store.Store's GetMetadata
func (s *ContextStoreWithDefault) GetMetadata(name string) (store.Metadata, error) {
if name == DefaultContextName {
defaultContext, err := s.Resolver()
if err != nil {
return store.Metadata{}, err
}
return defaultContext.Meta, nil
}
return s.Store.GetMetadata(name)
}
// ResetTLSMaterial is not implemented for default context and fails
func (s *ContextStoreWithDefault) ResetTLSMaterial(name string, data *store.ContextTLSData) error {
if name == DefaultContextName {
return invalidParameter(errors.New("default context cannot be edited"))
}
return s.Store.ResetTLSMaterial(name, data)
}
// ResetEndpointTLSMaterial is not implemented for default context and fails
func (s *ContextStoreWithDefault) ResetEndpointTLSMaterial(contextName string, endpointName string, data *store.EndpointTLSData) error {
if contextName == DefaultContextName {
return invalidParameter(errors.New("default context cannot be edited"))
}
return s.Store.ResetEndpointTLSMaterial(contextName, endpointName, data)
}
// ListTLSFiles implements store.Store's ListTLSFiles
func (s *ContextStoreWithDefault) ListTLSFiles(name string) (map[string]store.EndpointFiles, error) {
if name == DefaultContextName {
defaultContext, err := s.Resolver()
if err != nil {
return nil, err
}
tlsfiles := make(map[string]store.EndpointFiles)
for epName, epTLSData := range defaultContext.TLS.Endpoints {
var files store.EndpointFiles
for filename := range epTLSData.Files {
files = append(files, filename)
}
tlsfiles[epName] = files
}
return tlsfiles, nil
}
return s.Store.ListTLSFiles(name)
}
// GetTLSData implements store.Store's GetTLSData
func (s *ContextStoreWithDefault) GetTLSData(contextName, endpointName, fileName string) ([]byte, error) {
if contextName == DefaultContextName {
defaultContext, err := s.Resolver()
if err != nil {
return nil, err
}
if defaultContext.TLS.Endpoints[endpointName].Files[fileName] == nil {
return nil, notFound(errors.Errorf("TLS data for %s/%s/%s does not exist", DefaultContextName, endpointName, fileName))
}
return defaultContext.TLS.Endpoints[endpointName].Files[fileName], nil
}
return s.Store.GetTLSData(contextName, endpointName, fileName)
}
// GetStorageInfo implements store.Store's GetStorageInfo
func (s *ContextStoreWithDefault) GetStorageInfo(contextName string) store.StorageInfo {
if contextName == DefaultContextName {
return store.StorageInfo{MetadataPath: "<IN MEMORY>", TLSPath: "<IN MEMORY>"}
}
return s.Store.GetStorageInfo(contextName)
}

View File

@@ -0,0 +1,258 @@
package command
import (
"context"
"fmt"
"os"
"runtime"
"strings"
"github.com/distribution/reference"
"github.com/docker/cli/cli/config/configfile"
"github.com/docker/cli/cli/config/credentials"
configtypes "github.com/docker/cli/cli/config/types"
"github.com/docker/cli/cli/hints"
"github.com/docker/cli/cli/streams"
"github.com/docker/cli/internal/prompt"
"github.com/docker/cli/internal/tui"
registrytypes "github.com/docker/docker/api/types/registry"
"github.com/morikuni/aec"
"github.com/pkg/errors"
)
const (
registerSuggest = "Log in with your Docker ID or email address to push and pull images from Docker Hub. " +
"If you don't have a Docker ID, head over to https://hub.docker.com/ to create one."
patSuggest = "You can log in with your password or a Personal Access " +
"Token (PAT). Using a limited-scope PAT grants better security and is required " +
"for organizations using SSO. Learn more at https://docs.docker.com/go/access-tokens/"
)
// authConfigKey is the key used to store credentials for Docker Hub. It is
// a copy of [registry.IndexServer].
//
// [registry.IndexServer]: https://pkg.go.dev/github.com/docker/docker/registry#IndexServer
const authConfigKey = "https://index.docker.io/v1/"
// RegistryAuthenticationPrivilegedFunc returns a RequestPrivilegeFunc from the specified registry index info
// for the given command.
func RegistryAuthenticationPrivilegedFunc(cli Cli, index *registrytypes.IndexInfo, cmdName string) registrytypes.RequestAuthConfig {
configKey := getAuthConfigKey(index.Name)
isDefaultRegistry := configKey == authConfigKey || index.Official
return func(ctx context.Context) (string, error) {
_, _ = fmt.Fprintf(cli.Out(), "\nLogin prior to %s:\n", cmdName)
authConfig, err := GetDefaultAuthConfig(cli.ConfigFile(), true, configKey, isDefaultRegistry)
if err != nil {
_, _ = fmt.Fprintf(cli.Err(), "Unable to retrieve stored credentials for %s, error: %s.\n", authConfigKey, err)
}
select {
case <-ctx.Done():
return "", ctx.Err()
default:
}
authConfig, err = PromptUserForCredentials(ctx, cli, "", "", authConfig.Username, authConfigKey)
if err != nil {
return "", err
}
return registrytypes.EncodeAuthConfig(authConfig)
}
}
// ResolveAuthConfig returns auth-config for the given registry from the
// credential-store. It returns an empty AuthConfig if no credentials were
// found.
//
// It is similar to [registry.ResolveAuthConfig], but uses the credentials-
// store, instead of looking up credentials from a map.
func ResolveAuthConfig(cfg *configfile.ConfigFile, index *registrytypes.IndexInfo) registrytypes.AuthConfig {
configKey := index.Name
if index.Official {
configKey = authConfigKey
}
a, _ := cfg.GetAuthConfig(configKey)
return registrytypes.AuthConfig(a)
}
// GetDefaultAuthConfig gets the default auth config given a serverAddress
// If credentials for given serverAddress exists in the credential store, the configuration will be populated with values in it
func GetDefaultAuthConfig(cfg *configfile.ConfigFile, checkCredStore bool, serverAddress string, isDefaultRegistry bool) (registrytypes.AuthConfig, error) {
if !isDefaultRegistry {
serverAddress = credentials.ConvertToHostname(serverAddress)
}
authconfig := configtypes.AuthConfig{}
var err error
if checkCredStore {
authconfig, err = cfg.GetAuthConfig(serverAddress)
if err != nil {
return registrytypes.AuthConfig{
ServerAddress: serverAddress,
}, err
}
}
authconfig.ServerAddress = serverAddress
authconfig.IdentityToken = ""
return registrytypes.AuthConfig(authconfig), nil
}
// ConfigureAuth handles prompting of user's username and password if needed.
//
// Deprecated: use [PromptUserForCredentials] instead.
func ConfigureAuth(ctx context.Context, cli Cli, flUser, flPassword string, authConfig *registrytypes.AuthConfig, _ bool) error {
defaultUsername := authConfig.Username
serverAddress := authConfig.ServerAddress
newAuthConfig, err := PromptUserForCredentials(ctx, cli, flUser, flPassword, defaultUsername, serverAddress)
if err != nil {
return err
}
authConfig.Username = newAuthConfig.Username
authConfig.Password = newAuthConfig.Password
return nil
}
// PromptUserForCredentials handles the CLI prompt for the user to input
// credentials.
// If argUser is not empty, then the user is only prompted for their password.
// If argPassword is not empty, then the user is only prompted for their username
// If neither argUser nor argPassword are empty, then the user is not prompted and
// an AuthConfig is returned with those values.
// If defaultUsername is not empty, the username prompt includes that username
// and the user can hit enter without inputting a username to use that default
// username.
func PromptUserForCredentials(ctx context.Context, cli Cli, argUser, argPassword, defaultUsername, serverAddress string) (registrytypes.AuthConfig, error) {
// On Windows, force the use of the regular OS stdin stream.
//
// See:
// - https://github.com/moby/moby/issues/14336
// - https://github.com/moby/moby/issues/14210
// - https://github.com/moby/moby/pull/17738
//
// TODO(thaJeztah): we need to confirm if this special handling is still needed, as we may not be doing this in other places.
if runtime.GOOS == "windows" {
cli.SetIn(streams.NewIn(os.Stdin))
}
argUser = strings.TrimSpace(argUser)
if argUser == "" {
if serverAddress == authConfigKey {
// When signing in to the default (Docker Hub) registry, we display
// hints for creating an account, and (if hints are enabled), using
// a token instead of a password.
_, _ = fmt.Fprintln(cli.Out(), registerSuggest)
if hints.Enabled() {
_, _ = fmt.Fprintln(cli.Out(), patSuggest)
_, _ = fmt.Fprintln(cli.Out())
}
}
var msg string
defaultUsername = strings.TrimSpace(defaultUsername)
if defaultUsername == "" {
msg = "Username: "
} else {
msg = fmt.Sprintf("Username (%s): ", defaultUsername)
}
var err error
argUser, err = prompt.ReadInput(ctx, cli.In(), cli.Out(), msg)
if err != nil {
return registrytypes.AuthConfig{}, err
}
if argUser == "" {
argUser = defaultUsername
}
if argUser == "" {
return registrytypes.AuthConfig{}, errors.Errorf("Error: Non-null Username Required")
}
}
argPassword = strings.TrimSpace(argPassword)
if argPassword == "" {
restoreInput, err := prompt.DisableInputEcho(cli.In())
if err != nil {
return registrytypes.AuthConfig{}, err
}
defer func() {
if err := restoreInput(); err != nil {
// TODO(thaJeztah): we should consider printing instructions how
// to restore this manually (other than restarting the shell).
// e.g., 'run stty echo' when in a Linux or macOS shell, but
// PowerShell and CMD.exe may need different instructions.
_, _ = fmt.Fprintln(cli.Err(), "Error: failed to restore terminal state to echo input:", err)
}
}()
if serverAddress == authConfigKey {
out := tui.NewOutput(cli.Err())
out.PrintNote("A Personal Access Token (PAT) can be used instead.\n" +
"To create a PAT, visit " + aec.Underline.Apply("https://app.docker.com/settings") + "\n\n")
}
argPassword, err = prompt.ReadInput(ctx, cli.In(), cli.Out(), "Password: ")
if err != nil {
return registrytypes.AuthConfig{}, err
}
_, _ = fmt.Fprintln(cli.Out())
if argPassword == "" {
return registrytypes.AuthConfig{}, errors.Errorf("Error: Password Required")
}
}
return registrytypes.AuthConfig{
Username: argUser,
Password: argPassword,
ServerAddress: serverAddress,
}, nil
}
// RetrieveAuthTokenFromImage retrieves an encoded auth token given a complete
// image. The auth configuration is serialized as a base64url encoded RFC4648,
// section 5) JSON string for sending through the X-Registry-Auth header.
//
// For details on base64url encoding, see:
// - RFC4648, section 5: https://tools.ietf.org/html/rfc4648#section-5
func RetrieveAuthTokenFromImage(cfg *configfile.ConfigFile, image string) (string, error) {
// Retrieve encoded auth token from the image reference
authConfig, err := resolveAuthConfigFromImage(cfg, image)
if err != nil {
return "", err
}
encodedAuth, err := registrytypes.EncodeAuthConfig(authConfig)
if err != nil {
return "", err
}
return encodedAuth, nil
}
// resolveAuthConfigFromImage retrieves that AuthConfig using the image string
func resolveAuthConfigFromImage(cfg *configfile.ConfigFile, image string) (registrytypes.AuthConfig, error) {
registryRef, err := reference.ParseNormalizedNamed(image)
if err != nil {
return registrytypes.AuthConfig{}, err
}
configKey := getAuthConfigKey(reference.Domain(registryRef))
a, err := cfg.GetAuthConfig(configKey)
if err != nil {
return registrytypes.AuthConfig{}, err
}
return registrytypes.AuthConfig(a), nil
}
// getAuthConfigKey special-cases using the full index address of the official
// index as the AuthConfig key, and uses the (host)name[:port] for private indexes.
//
// It is similar to [registry.GetAuthConfigKey], but does not require on
// [registrytypes.IndexInfo] as intermediate.
//
// [registry.GetAuthConfigKey]: https://pkg.go.dev/github.com/docker/docker/registry#GetAuthConfigKey
// [registrytypes.IndexInfo]:https://pkg.go.dev/github.com/docker/docker/api/types/registry#IndexInfo
func getAuthConfigKey(domainName string) string {
if domainName == "docker.io" || domainName == "index.docker.io" {
return authConfigKey
}
return domainName
}

View File

@@ -0,0 +1,265 @@
package command
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/google/uuid"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
"go.opentelemetry.io/otel/trace"
)
const exportTimeout = 50 * time.Millisecond
// TracerProvider is an extension of the trace.TracerProvider interface for CLI programs.
type TracerProvider interface {
trace.TracerProvider
ForceFlush(ctx context.Context) error
Shutdown(ctx context.Context) error
}
// MeterProvider is an extension of the metric.MeterProvider interface for CLI programs.
type MeterProvider interface {
metric.MeterProvider
ForceFlush(ctx context.Context) error
Shutdown(ctx context.Context) error
}
// TelemetryClient provides the methods for using OTEL tracing or metrics.
type TelemetryClient interface {
// Resource returns the OTEL Resource configured with this TelemetryClient.
// This resource may be created lazily, but the resource should be the same
// each time this function is invoked.
Resource() *resource.Resource
// TracerProvider returns the currently initialized TracerProvider. This TracerProvider will be configured
// with the default tracing components for a CLI program
TracerProvider() trace.TracerProvider
// MeterProvider returns the currently initialized MeterProvider. This MeterProvider will be configured
// with the default metric components for a CLI program
MeterProvider() metric.MeterProvider
}
func (cli *DockerCli) Resource() *resource.Resource {
return cli.res.Get()
}
func (*DockerCli) TracerProvider() trace.TracerProvider {
return otel.GetTracerProvider()
}
func (*DockerCli) MeterProvider() metric.MeterProvider {
return otel.GetMeterProvider()
}
// WithResourceOptions configures additional options for the default resource. The default
// resource will continue to include its default options.
func WithResourceOptions(opts ...resource.Option) CLIOption {
return func(cli *DockerCli) error {
cli.res.AppendOptions(opts...)
return nil
}
}
// WithResource overwrites the default resource and prevents its creation.
func WithResource(res *resource.Resource) CLIOption {
return func(cli *DockerCli) error {
cli.res.Set(res)
return nil
}
}
type telemetryResource struct {
res *resource.Resource
opts []resource.Option
once sync.Once
}
func (r *telemetryResource) Set(res *resource.Resource) {
r.res = res
}
func (r *telemetryResource) Get() *resource.Resource {
r.once.Do(r.init)
return r.res
}
func (r *telemetryResource) init() {
if r.res != nil {
r.opts = nil
return
}
opts := append(defaultResourceOptions(), r.opts...)
res, err := resource.New(context.Background(), opts...)
if err != nil {
otel.Handle(err)
}
r.res = res
// Clear the resource options since they'll never be used again and to allow
// the garbage collector to retrieve that memory.
r.opts = nil
}
// createGlobalMeterProvider creates a new MeterProvider from the initialized DockerCli struct
// with the given options and sets it as the global meter provider
func (cli *DockerCli) createGlobalMeterProvider(ctx context.Context, opts ...sdkmetric.Option) {
allOpts := make([]sdkmetric.Option, 0, len(opts)+2)
allOpts = append(allOpts, sdkmetric.WithResource(cli.Resource()))
allOpts = append(allOpts, dockerMetricExporter(ctx, cli)...)
allOpts = append(allOpts, opts...)
mp := sdkmetric.NewMeterProvider(allOpts...)
otel.SetMeterProvider(mp)
}
// createGlobalTracerProvider creates a new TracerProvider from the initialized DockerCli struct
// with the given options and sets it as the global tracer provider
func (cli *DockerCli) createGlobalTracerProvider(ctx context.Context, opts ...sdktrace.TracerProviderOption) {
allOpts := make([]sdktrace.TracerProviderOption, 0, len(opts)+2)
allOpts = append(allOpts, sdktrace.WithResource(cli.Resource()))
allOpts = append(allOpts, dockerSpanExporter(ctx, cli)...)
allOpts = append(allOpts, opts...)
tp := sdktrace.NewTracerProvider(allOpts...)
otel.SetTracerProvider(tp)
}
func defaultResourceOptions() []resource.Option {
return []resource.Option{
resource.WithDetectors(serviceNameDetector{}),
resource.WithAttributes(
// Use a unique instance id so OTEL knows that each invocation
// of the CLI is its own instance. Without this, downstream
// OTEL processors may think the same process is restarting
// continuously.
semconv.ServiceInstanceID(uuid.NewString()),
),
resource.WithFromEnv(),
resource.WithTelemetrySDK(),
}
}
func (r *telemetryResource) AppendOptions(opts ...resource.Option) {
if r.res != nil {
return
}
r.opts = append(r.opts, opts...)
}
type serviceNameDetector struct{}
func (serviceNameDetector) Detect(ctx context.Context) (*resource.Resource, error) {
return resource.StringDetector(
semconv.SchemaURL,
semconv.ServiceNameKey,
func() (string, error) {
return filepath.Base(os.Args[0]), nil
},
).Detect(ctx)
}
// cliReader is an implementation of Reader that will automatically
// report to a designated Exporter when Shutdown is called.
type cliReader struct {
sdkmetric.Reader
exporter sdkmetric.Exporter
}
func newCLIReader(exp sdkmetric.Exporter) sdkmetric.Reader {
reader := sdkmetric.NewManualReader(
sdkmetric.WithTemporalitySelector(deltaTemporality),
)
return &cliReader{
Reader: reader,
exporter: exp,
}
}
func (r *cliReader) Shutdown(ctx context.Context) error {
// Place a pretty tight constraint on the actual reporting.
// We don't want CLI metrics to prevent the CLI from exiting
// so if there's some kind of issue we need to abort pretty
// quickly.
ctx, cancel := context.WithTimeout(ctx, exportTimeout)
defer cancel()
return r.ForceFlush(ctx)
}
func (r *cliReader) ForceFlush(ctx context.Context) error {
var rm metricdata.ResourceMetrics
if err := r.Reader.Collect(ctx, &rm); err != nil {
return err
}
return r.exporter.Export(ctx, &rm)
}
// deltaTemporality sets the Temporality of every instrument to delta.
//
// This isn't really needed since we create a unique resource on each invocation,
// but it can help with cardinality concerns for downstream processors since they can
// perform aggregation for a time interval and then discard the data once that time
// period has passed. Cumulative temporality would imply to the downstream processor
// that they might receive a successive point and they may unnecessarily keep state
// they really shouldn't.
func deltaTemporality(_ sdkmetric.InstrumentKind) metricdata.Temporality {
return metricdata.DeltaTemporality
}
// resourceAttributesEnvVar is the name of the envvar that includes additional
// resource attributes for OTEL as defined in the [OpenTelemetry specification].
//
// [OpenTelemetry specification]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration
const resourceAttributesEnvVar = "OTEL_RESOURCE_ATTRIBUTES"
func filterResourceAttributesEnvvar() {
if v := os.Getenv(resourceAttributesEnvVar); v != "" {
if filtered := filterResourceAttributes(v); filtered != "" {
_ = os.Setenv(resourceAttributesEnvVar, filtered)
} else {
_ = os.Unsetenv(resourceAttributesEnvVar)
}
}
}
// dockerCLIAttributePrefix is the prefix for any docker cli OTEL attributes.
// When updating, make sure to also update the copy in cli-plugins/manager.
//
// TODO(thaJeztah): move telemetry-related code to an (internal) package to reduce dependency on cli/command in cli-plugins, which has too many imports.
const dockerCLIAttributePrefix = "docker.cli."
func filterResourceAttributes(s string) string {
if trimmed := strings.TrimSpace(s); trimmed == "" {
return trimmed
}
pairs := strings.Split(s, ",")
elems := make([]string, 0, len(pairs))
for _, p := range pairs {
k, _, found := strings.Cut(p, "=")
if !found {
// Do not interact with invalid otel resources.
elems = append(elems, p)
continue
}
// Skip attributes that have our docker.cli prefix.
if strings.HasPrefix(k, dockerCLIAttributePrefix) {
continue
}
elems = append(elems, p)
}
return strings.Join(elems, ",")
}

View File

@@ -0,0 +1,241 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package command
import (
"context"
"fmt"
"io/fs"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"unicode"
"github.com/pkg/errors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
const (
otelContextFieldName string = "otel"
otelExporterOTLPEndpoint string = "OTEL_EXPORTER_OTLP_ENDPOINT"
debugEnvVarPrefix string = "DOCKER_CLI_"
)
// dockerExporterOTLPEndpoint retrieves the OTLP endpoint used for the docker reporter
// from the current context.
func dockerExporterOTLPEndpoint(cli Cli) (endpoint string, secure bool) {
meta, err := cli.ContextStore().GetMetadata(cli.CurrentContext())
if err != nil {
otel.Handle(err)
return "", false
}
var otelCfg any
switch m := meta.Metadata.(type) {
case DockerContext:
otelCfg = m.AdditionalFields[otelContextFieldName]
case map[string]any:
otelCfg = m[otelContextFieldName]
}
if otelCfg != nil {
otelMap, ok := otelCfg.(map[string]any)
if !ok {
otel.Handle(errors.Errorf(
"unexpected type for field %q: %T (expected: %T)",
otelContextFieldName,
otelCfg,
otelMap,
))
}
// keys from https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/
endpoint, _ = otelMap[otelExporterOTLPEndpoint].(string)
}
// Override with env var value if it exists AND IS SET
// (ignore otel defaults for this override when the key exists but is empty)
if override := os.Getenv(debugEnvVarPrefix + otelExporterOTLPEndpoint); override != "" {
endpoint = override
}
if endpoint == "" {
return "", false
}
// Parse the endpoint. The docker config expects the endpoint to be
// in the form of a URL to match the environment variable, but this
// option doesn't correspond directly to WithEndpoint.
//
// We pretend we're the same as the environment reader.
u, err := url.Parse(endpoint)
if err != nil {
otel.Handle(errors.Errorf("docker otel endpoint is invalid: %s", err))
return "", false
}
switch u.Scheme {
case "unix":
endpoint = unixSocketEndpoint(u)
case "https":
secure = true
fallthrough
case "http":
endpoint = path.Join(u.Host, u.Path)
}
return endpoint, secure
}
func dockerSpanExporter(ctx context.Context, cli Cli) []sdktrace.TracerProviderOption {
endpoint, secure := dockerExporterOTLPEndpoint(cli)
if endpoint == "" {
return nil
}
opts := []otlptracegrpc.Option{
otlptracegrpc.WithEndpoint(endpoint),
}
if !secure {
opts = append(opts, otlptracegrpc.WithInsecure())
}
exp, err := otlptracegrpc.New(ctx, opts...)
if err != nil {
otel.Handle(err)
return nil
}
return []sdktrace.TracerProviderOption{sdktrace.WithBatcher(exp, sdktrace.WithExportTimeout(exportTimeout))}
}
func dockerMetricExporter(ctx context.Context, cli Cli) []sdkmetric.Option {
endpoint, secure := dockerExporterOTLPEndpoint(cli)
if endpoint == "" {
return nil
}
opts := []otlpmetricgrpc.Option{
otlpmetricgrpc.WithEndpoint(endpoint),
}
if !secure {
opts = append(opts, otlpmetricgrpc.WithInsecure())
}
exp, err := otlpmetricgrpc.New(ctx, opts...)
if err != nil {
otel.Handle(err)
return nil
}
return []sdkmetric.Option{sdkmetric.WithReader(newCLIReader(exp))}
}
// unixSocketEndpoint converts the unix scheme from URL to
// an OTEL endpoint that can be used with the OTLP exporter.
//
// The OTLP exporter handles unix sockets in a strange way.
// It seems to imply they can be used as an environment variable
// and are handled properly, but they don't seem to be as the behavior
// of the environment variable is to strip the scheme from the endpoint
// while the underlying implementation needs the scheme to use the
// correct resolver.
func unixSocketEndpoint(u *url.URL) string {
// GRPC does not allow host to be used.
socketPath := u.Path
// If we are on windows and we have an absolute path
// that references a letter drive, check to see if the
// WSL equivalent path exists and we should use that instead.
if isWsl() {
if p := wslSocketPath(socketPath, os.DirFS("/")); p != "" {
socketPath = p
}
}
// Enforce that we are using forward slashes.
return "unix://" + filepath.ToSlash(socketPath)
}
// wslSocketPath will convert the referenced URL to a WSL-compatible
// path and check if that path exists. If the path exists, it will
// be returned.
func wslSocketPath(s string, f fs.FS) string {
if p := toWslPath(s); p != "" {
if _, err := stat(p, f); err == nil {
return "/" + p
}
}
return ""
}
// toWslPath converts the referenced URL to a WSL-compatible
// path if this looks like a Windows absolute path.
//
// If no drive is in the URL, defaults to the C drive.
func toWslPath(s string) string {
drive, p, ok := parseUNCPath(s)
if !ok {
return ""
}
return fmt.Sprintf("mnt/%s%s", strings.ToLower(drive), p)
}
func parseUNCPath(s string) (drive, p string, ok bool) {
// UNC paths use backslashes but we're using forward slashes
// so also enforce that here.
//
// In reality, this should have been enforced much earlier
// than here since backslashes aren't allowed in URLs, but
// we're going to code defensively here.
s = filepath.ToSlash(s)
const uncPrefix = "//./"
if !strings.HasPrefix(s, uncPrefix) {
// Not a UNC path.
return "", "", false
}
s = s[len(uncPrefix):]
parts := strings.SplitN(s, "/", 2)
if len(parts) != 2 {
// Not enough components.
return "", "", false
}
drive, ok = splitWindowsDrive(parts[0])
if !ok {
// Not a windows drive.
return "", "", false
}
return drive, "/" + parts[1], true
}
// splitWindowsDrive checks if the string references a windows
// drive (such as c:) and returns the drive letter if it is.
func splitWindowsDrive(s string) (string, bool) {
if b := []rune(s); len(b) == 2 && unicode.IsLetter(b[0]) && b[1] == ':' {
return string(b[0]), true
}
return "", false
}
func stat(p string, f fs.FS) (fs.FileInfo, error) {
if f, ok := f.(fs.StatFS); ok {
return f.Stat(p)
}
file, err := f.Open(p)
if err != nil {
return nil, err
}
defer file.Close()
return file.Stat()
}
func isWsl() bool {
return os.Getenv("WSL_DISTRO_NAME") != ""
}

View File

@@ -0,0 +1,25 @@
package command
// WithEnableGlobalMeterProvider configures the DockerCli to create a new
// MeterProvider from the initialized DockerCli struct, and set it as
// the global meter provider.
//
// WARNING: For internal use, don't depend on this.
func WithEnableGlobalMeterProvider() CLIOption {
return func(cli *DockerCli) error {
cli.enableGlobalMeter = true
return nil
}
}
// WithEnableGlobalTracerProvider configures the DockerCli to create a new
// TracerProvider from the initialized DockerCli struct, and set it as
// the global tracer provider.
//
// WARNING: For internal use, don't depend on this.
func WithEnableGlobalTracerProvider() CLIOption {
return func(cli *DockerCli) error {
cli.enableGlobalTracer = true
return nil
}
}

View File

@@ -0,0 +1,182 @@
package command
import (
"context"
"fmt"
"strings"
"time"
"github.com/docker/cli/cli/version"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
// BaseCommandAttributes returns an attribute.Set containing attributes to attach to metrics/traces
func BaseCommandAttributes(cmd *cobra.Command, streams Streams) []attribute.KeyValue {
return append([]attribute.KeyValue{
attribute.String("command.name", getCommandName(cmd)),
}, stdioAttributes(streams)...)
}
// InstrumentCobraCommands wraps all cobra commands' RunE funcs to set a command duration metric using otel.
//
// Note: this should be the last func to wrap/modify the PersistentRunE/RunE funcs before command execution.
//
// can also be used for spans!
func (cli *DockerCli) InstrumentCobraCommands(ctx context.Context, cmd *cobra.Command) {
// If PersistentPreRunE is nil, make it execute PersistentPreRun and return nil by default
ogPersistentPreRunE := cmd.PersistentPreRunE
if ogPersistentPreRunE == nil {
ogPersistentPreRun := cmd.PersistentPreRun
//nolint:unparam // necessary because error will always be nil here
ogPersistentPreRunE = func(cmd *cobra.Command, args []string) error {
ogPersistentPreRun(cmd, args)
return nil
}
cmd.PersistentPreRun = nil
}
// wrap RunE in PersistentPreRunE so that this operation gets executed on all children commands
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
// If RunE is nil, make it execute Run and return nil by default
ogRunE := cmd.RunE
if ogRunE == nil {
ogRun := cmd.Run
//nolint:unparam // necessary because error will always be nil here
ogRunE = func(cmd *cobra.Command, args []string) error {
ogRun(cmd, args)
return nil
}
cmd.Run = nil
}
cmd.RunE = func(cmd *cobra.Command, args []string) error {
// start the timer as the first step of every cobra command
stopInstrumentation := cli.StartInstrumentation(cmd)
cmdErr := ogRunE(cmd, args)
stopInstrumentation(cmdErr)
return cmdErr
}
return ogPersistentPreRunE(cmd, args)
}
}
// StartInstrumentation instruments CLI commands with the individual metrics and spans configured.
// It's the main command OTel utility, and new command-related metrics should be added to it.
// It should be called immediately before command execution, and returns a stopInstrumentation function
// that must be called with the error resulting from the command execution.
func (cli *DockerCli) StartInstrumentation(cmd *cobra.Command) (stopInstrumentation func(error)) {
baseAttrs := BaseCommandAttributes(cmd, cli)
return startCobraCommandTimer(cli.MeterProvider(), baseAttrs)
}
func startCobraCommandTimer(mp metric.MeterProvider, attrs []attribute.KeyValue) func(err error) {
meter := getDefaultMeter(mp)
durationCounter, _ := meter.Float64Counter(
"command.time",
metric.WithDescription("Measures the duration of the cobra command"),
metric.WithUnit("ms"),
)
start := time.Now()
return func(err error) {
// Use a new context for the export so that the command being cancelled
// doesn't affect the metrics, and we get metrics for cancelled commands.
ctx, cancel := context.WithTimeout(context.Background(), exportTimeout)
defer cancel()
duration := float64(time.Since(start)) / float64(time.Millisecond)
cmdStatusAttrs := attributesFromError(err)
durationCounter.Add(ctx, duration,
metric.WithAttributes(attrs...),
metric.WithAttributes(cmdStatusAttrs...),
)
if mp, ok := mp.(MeterProvider); ok {
if err := mp.ForceFlush(ctx); err != nil {
otel.Handle(err)
}
}
}
}
func stdioAttributes(streams Streams) []attribute.KeyValue {
return []attribute.KeyValue{
attribute.Bool("command.stdin.isatty", streams.In().IsTerminal()),
attribute.Bool("command.stdout.isatty", streams.Out().IsTerminal()),
attribute.Bool("command.stderr.isatty", streams.Err().IsTerminal()),
}
}
func attributesFromError(err error) []attribute.KeyValue {
attrs := []attribute.KeyValue{}
exitCode := 0
if err != nil {
exitCode = 1
if stderr, ok := err.(statusError); ok {
// StatusError should only be used for errors, and all errors should
// have a non-zero exit status, so only set this here if this value isn't 0
if stderr.StatusCode != 0 {
exitCode = stderr.StatusCode
}
}
attrs = append(attrs, attribute.String("command.error.type", otelErrorType(err)))
}
attrs = append(attrs, attribute.Int("command.status.code", exitCode))
return attrs
}
// otelErrorType returns an attribute for the error type based on the error category.
func otelErrorType(err error) string {
name := "generic"
if errors.Is(err, context.Canceled) {
name = "canceled"
}
return name
}
// statusError reports an unsuccessful exit by a command.
type statusError struct {
Status string
StatusCode int
}
func (e statusError) Error() string {
return fmt.Sprintf("Status: %s, Code: %d", e.Status, e.StatusCode)
}
// getCommandName gets the cobra command name in the format
// `... parentCommandName commandName` by traversing it's parent commands recursively.
// until the root command is reached.
//
// Note: The root command's name is excluded. If cmd is the root cmd, return ""
func getCommandName(cmd *cobra.Command) string {
fullCmdName := getFullCommandName(cmd)
i := strings.Index(fullCmdName, " ")
if i == -1 {
return ""
}
return fullCmdName[i+1:]
}
// getFullCommandName gets the full cobra command name in the format
// `... parentCommandName commandName` by traversing it's parent commands recursively
// until the root command is reached.
func getFullCommandName(cmd *cobra.Command) string {
if cmd.HasParent() {
return fmt.Sprintf("%s %s", getFullCommandName(cmd.Parent()), cmd.Name())
}
return cmd.Name()
}
// getDefaultMeter gets the default metric.Meter for the application
// using the given metric.MeterProvider
func getDefaultMeter(mp metric.MeterProvider) metric.Meter {
return mp.Meter(
"github.com/docker/cli",
metric.WithInstrumentationVersion(version.Version),
)
}

View File

@@ -0,0 +1,15 @@
package command
import (
"github.com/spf13/pflag"
)
// AddTrustVerificationFlags adds content trust flags to the provided flagset
func AddTrustVerificationFlags(fs *pflag.FlagSet, v *bool, trusted bool) {
fs.BoolVar(v, "disable-content-trust", !trusted, "Skip image verification")
}
// AddTrustSigningFlags adds "signing" flags to the provided flagset
func AddTrustSigningFlags(fs *pflag.FlagSet, v *bool, trusted bool) {
fs.BoolVar(v, "disable-content-trust", !trusted, "Skip image signing")
}

View File

@@ -0,0 +1,172 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package command
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/streams"
"github.com/docker/cli/internal/prompt"
"github.com/docker/docker/api/types/filters"
"github.com/moby/sys/atomicwriter"
"github.com/pkg/errors"
"github.com/spf13/pflag"
)
// CopyToFile writes the content of the reader to the specified file
//
// Deprecated: use [atomicwriter.New].
func CopyToFile(outfile string, r io.Reader) error {
writer, err := atomicwriter.New(outfile, 0o600)
if err != nil {
return err
}
defer writer.Close()
_, err = io.Copy(writer, r)
return err
}
const ErrPromptTerminated = prompt.ErrTerminated
// DisableInputEcho disables input echo on the provided streams.In.
// This is useful when the user provides sensitive information like passwords.
// The function returns a restore function that should be called to restore the
// terminal state.
func DisableInputEcho(ins *streams.In) (restore func() error, err error) {
return prompt.DisableInputEcho(ins)
}
// PromptForInput requests input from the user.
//
// If the user terminates the CLI with SIGINT or SIGTERM while the prompt is
// active, the prompt will return an empty string ("") with an ErrPromptTerminated error.
// When the prompt returns an error, the caller should propagate the error up
// the stack and close the io.Reader used for the prompt which will prevent the
// background goroutine from blocking indefinitely.
func PromptForInput(ctx context.Context, in io.Reader, out io.Writer, message string) (string, error) {
return prompt.ReadInput(ctx, in, out, message)
}
// PromptForConfirmation requests and checks confirmation from the user.
// This will display the provided message followed by ' [y/N] '. If the user
// input 'y' or 'Y' it returns true otherwise false. If no message is provided,
// "Are you sure you want to proceed? [y/N] " will be used instead.
//
// If the user terminates the CLI with SIGINT or SIGTERM while the prompt is
// active, the prompt will return false with an ErrPromptTerminated error.
// When the prompt returns an error, the caller should propagate the error up
// the stack and close the io.Reader used for the prompt which will prevent the
// background goroutine from blocking indefinitely.
func PromptForConfirmation(ctx context.Context, ins io.Reader, outs io.Writer, message string) (bool, error) {
return prompt.Confirm(ctx, ins, outs, message)
}
// PruneFilters merges prune filters specified in config.json with those specified
// as command-line flags.
//
// CLI label filters have precedence over those specified in config.json. If a
// label filter specified as flag conflicts with a label defined in config.json
// (i.e., "label=some-value" conflicts with "label!=some-value", and vice versa),
// then the filter defined in config.json is omitted.
func PruneFilters(dockerCLI config.Provider, pruneFilters filters.Args) filters.Args {
cfg := dockerCLI.ConfigFile()
if cfg == nil {
return pruneFilters
}
// Merge filters provided through the CLI with default filters defined
// in the CLI-configfile.
for _, f := range cfg.PruneFilters {
k, v, ok := strings.Cut(f, "=")
if !ok {
continue
}
switch k {
case "label":
// "label != some-value" conflicts with "label = some-value"
if pruneFilters.ExactMatch("label!", v) {
continue
}
pruneFilters.Add(k, v)
case "label!":
// "label != some-value" conflicts with "label = some-value"
if pruneFilters.ExactMatch("label", v) {
continue
}
pruneFilters.Add(k, v)
default:
pruneFilters.Add(k, v)
}
}
return pruneFilters
}
// AddPlatformFlag adds `platform` to a set of flags for API version 1.32 and later.
func AddPlatformFlag(flags *pflag.FlagSet, target *string) {
flags.StringVar(target, "platform", os.Getenv("DOCKER_DEFAULT_PLATFORM"), "Set platform if server is multi-platform capable")
_ = flags.SetAnnotation("platform", "version", []string{"1.32"})
}
// ValidateOutputPath validates the output paths of the "docker cp" command.
func ValidateOutputPath(path string) error {
dir := filepath.Dir(filepath.Clean(path))
if dir != "" && dir != "." {
if _, err := os.Stat(dir); os.IsNotExist(err) {
return errors.Errorf("invalid output path: directory %q does not exist", dir)
}
}
// check whether `path` points to a regular file
// (if the path exists and doesn't point to a directory)
if fileInfo, err := os.Stat(path); !os.IsNotExist(err) {
if err != nil {
return err
}
if fileInfo.Mode().IsDir() || fileInfo.Mode().IsRegular() {
return nil
}
if err := ValidateOutputPathFileMode(fileInfo.Mode()); err != nil {
return errors.Wrapf(err, "invalid output path: %q must be a directory or a regular file", path)
}
}
return nil
}
// ValidateOutputPathFileMode validates the output paths of the "docker cp" command
// and serves as a helper to [ValidateOutputPath]
func ValidateOutputPathFileMode(fileMode os.FileMode) error {
switch {
case fileMode&os.ModeDevice != 0:
return errors.New("got a device")
case fileMode&os.ModeIrregular != 0:
return errors.New("got an irregular file")
}
return nil
}
func invalidParameter(err error) error {
return invalidParameterErr{err}
}
type invalidParameterErr struct{ error }
func (invalidParameterErr) InvalidParameter() {}
func notFound(err error) error {
return notFoundErr{err}
}
type notFoundErr struct{ error }
func (notFoundErr) NotFound() {}
func (e notFoundErr) Unwrap() error {
return e.error
}

View File

@@ -0,0 +1,6 @@
package docker
const (
// DockerEndpoint is the name of the docker endpoint in a stored context
DockerEndpoint = "docker"
)

View File

@@ -0,0 +1,166 @@
package docker
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"net"
"net/http"
"strings"
"time"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/cli/cli/context"
"github.com/docker/cli/cli/context/store"
"github.com/docker/docker/client"
"github.com/docker/go-connections/tlsconfig"
"github.com/pkg/errors"
)
// EndpointMeta is a typed wrapper around a context-store generic endpoint describing
// a Docker Engine endpoint, without its tls config
type EndpointMeta = context.EndpointMetaBase
// Endpoint is a typed wrapper around a context-store generic endpoint describing
// a Docker Engine endpoint, with its tls data
type Endpoint struct {
EndpointMeta
TLSData *context.TLSData
}
// WithTLSData loads TLS materials for the endpoint
func WithTLSData(s store.Reader, contextName string, m EndpointMeta) (Endpoint, error) {
tlsData, err := context.LoadTLSData(s, contextName, DockerEndpoint)
if err != nil {
return Endpoint{}, err
}
return Endpoint{
EndpointMeta: m,
TLSData: tlsData,
}, nil
}
// tlsConfig extracts a context docker endpoint TLS config
func (ep *Endpoint) tlsConfig() (*tls.Config, error) {
if ep.TLSData == nil && !ep.SkipTLSVerify {
// there is no specific tls config
return nil, nil
}
var tlsOpts []func(*tls.Config)
if ep.TLSData != nil && ep.TLSData.CA != nil {
certPool := x509.NewCertPool()
if !certPool.AppendCertsFromPEM(ep.TLSData.CA) {
return nil, errors.New("failed to retrieve context tls info: ca.pem seems invalid")
}
tlsOpts = append(tlsOpts, func(cfg *tls.Config) {
cfg.RootCAs = certPool
})
}
if ep.TLSData != nil && ep.TLSData.Key != nil && ep.TLSData.Cert != nil {
keyBytes := ep.TLSData.Key
pemBlock, _ := pem.Decode(keyBytes)
if pemBlock == nil {
return nil, errors.New("no valid private key found")
}
if x509.IsEncryptedPEMBlock(pemBlock) { //nolint:staticcheck // SA1019: x509.IsEncryptedPEMBlock is deprecated, and insecure by design
return nil, errors.New("private key is encrypted - support for encrypted private keys has been removed, see https://docs.docker.com/go/deprecated/")
}
x509cert, err := tls.X509KeyPair(ep.TLSData.Cert, keyBytes)
if err != nil {
return nil, errors.Wrap(err, "failed to retrieve context tls info")
}
tlsOpts = append(tlsOpts, func(cfg *tls.Config) {
cfg.Certificates = []tls.Certificate{x509cert}
})
}
if ep.SkipTLSVerify {
tlsOpts = append(tlsOpts, func(cfg *tls.Config) {
cfg.InsecureSkipVerify = true
})
}
return tlsconfig.ClientDefault(tlsOpts...), nil
}
// ClientOpts returns a slice of Client options to configure an API client with this endpoint
func (ep *Endpoint) ClientOpts() ([]client.Opt, error) {
var result []client.Opt
if ep.Host != "" {
helper, err := connhelper.GetConnectionHelper(ep.Host)
if err != nil {
return nil, err
}
if helper == nil {
// Check if we're connecting over a socket, because there's no
// need to configure TLS for a socket connection.
//
// TODO(thaJeztah); make resolveDockerEndpoint and resolveDefaultDockerEndpoint not load TLS data,
// and load TLS files lazily; see https://github.com/docker/cli/pull/1581
if !isSocket(ep.Host) {
tlsConfig, err := ep.tlsConfig()
if err != nil {
return nil, err
}
result = append(result, withHTTPClient(tlsConfig))
}
result = append(result, client.WithHost(ep.Host))
} else {
result = append(result,
client.WithHTTPClient(&http.Client{
// No TLS, and no proxy.
Transport: &http.Transport{
DialContext: helper.Dialer,
},
}),
client.WithHost(helper.Host),
client.WithDialContext(helper.Dialer),
)
}
}
result = append(result, client.WithVersionFromEnv(), client.WithAPIVersionNegotiation())
return result, nil
}
// isSocket checks if the given address is a Unix-socket (linux),
// named pipe (Windows), or file-descriptor.
func isSocket(addr string) bool {
switch proto, _, _ := strings.Cut(addr, "://"); proto {
case "unix", "npipe", "fd":
return true
default:
return false
}
}
func withHTTPClient(tlsConfig *tls.Config) func(*client.Client) error {
return func(c *client.Client) error {
if tlsConfig == nil {
// Use the default HTTPClient
return nil
}
return client.WithHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsConfig,
DialContext: (&net.Dialer{
KeepAlive: 30 * time.Second,
Timeout: 30 * time.Second,
}).DialContext,
},
CheckRedirect: client.CheckRedirect,
})(c)
}
}
// EndpointFromContext parses a context docker endpoint metadata into a typed EndpointMeta structure
func EndpointFromContext(metadata store.Metadata) (EndpointMeta, error) {
ep, ok := metadata.Endpoints[DockerEndpoint]
if !ok {
return EndpointMeta{}, errors.New("cannot find docker endpoint in context")
}
typed, ok := ep.(EndpointMeta)
if !ok {
return EndpointMeta{}, errors.Errorf("endpoint %q is not of type EndpointMeta", DockerEndpoint)
}
return typed, nil
}

View File

@@ -0,0 +1,7 @@
package context
// EndpointMetaBase contains fields we expect to be common for most context endpoints
type EndpointMetaBase struct {
Host string `json:",omitempty"`
SkipTLSVerify bool
}

View File

@@ -0,0 +1,32 @@
// Package store provides a generic way to store credentials to connect to
// virtually any kind of remote system.
// The term `context` comes from the similar feature in Kubernetes kubectl
// config files.
//
// Conceptually, a context is a set of metadata and TLS data, that can be used
// to connect to various endpoints of a remote system. TLS data and metadata
// are stored separately, so that in the future, we will be able to store
// sensitive information in a more secure way, depending on the os we are running
// on (e.g.: on Windows we could use the user Certificate Store, on macOS the
// user Keychain...).
//
// Current implementation is purely file based with the following structure:
//
// ${CONTEXT_ROOT}
// meta/
// <context id>/meta.json: contains context medata (key/value pairs) as
// well as a list of endpoints (themselves containing
// key/value pair metadata).
// tls/
// <context id>/endpoint1/: directory containing TLS data for the endpoint1
// in the corresponding context.
//
// The context store itself has absolutely no knowledge about what a docker
// endpoint should contain in term of metadata or TLS config. Client code is
// responsible for generating and parsing endpoint metadata and TLS files. The
// multi-endpoints approach of this package allows to combine many different
// endpoints in the same "context".
//
// Context IDs are actually SHA256 hashes of the context name, and are there
// only to avoid dealing with special characters in context names.
package store

View File

@@ -0,0 +1,28 @@
package store
import cerrdefs "github.com/containerd/errdefs"
func invalidParameter(err error) error {
if err == nil || cerrdefs.IsInvalidArgument(err) {
return err
}
return invalidParameterErr{err}
}
type invalidParameterErr struct{ error }
func (invalidParameterErr) InvalidParameter() {}
func notFound(err error) error {
if err == nil || cerrdefs.IsNotFound(err) {
return err
}
return notFoundErr{err}
}
type notFoundErr struct{ error }
func (notFoundErr) NotFound() {}
func (e notFoundErr) Unwrap() error {
return e.error
}

View File

@@ -0,0 +1,29 @@
package store
import (
"errors"
"io"
)
// limitedReader is a fork of [io.LimitedReader] to override Read.
type limitedReader struct {
R io.Reader
N int64 // max bytes remaining
}
// Read is a fork of [io.LimitedReader.Read] that returns an error when limit exceeded.
func (l *limitedReader) Read(p []byte) (n int, err error) {
if l.N < 0 {
return 0, errors.New("read exceeds the defined limit")
}
if l.N == 0 {
return 0, io.EOF
}
// have to cap N + 1 otherwise we won't hit limit err
if int64(len(p)) > l.N+1 {
p = p[0 : l.N+1]
}
n, err = l.R.Read(p)
l.N -= int64(n)
return n, err
}

View File

@@ -0,0 +1,166 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package store
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"github.com/fvbommel/sortorder"
"github.com/moby/sys/atomicwriter"
)
const (
metadataDir = "meta"
metaFile = "meta.json"
)
type metadataStore struct {
root string
config Config
}
func (s *metadataStore) contextDir(id contextdir) string {
return filepath.Join(s.root, string(id))
}
func (s *metadataStore) createOrUpdate(meta Metadata) error {
contextDir := s.contextDir(contextdirOf(meta.Name))
if err := os.MkdirAll(contextDir, 0o755); err != nil {
return err
}
bytes, err := json.Marshal(&meta)
if err != nil {
return err
}
return atomicwriter.WriteFile(filepath.Join(contextDir, metaFile), bytes, 0o644)
}
func parseTypedOrMap(payload []byte, getter TypeGetter) (any, error) {
if len(payload) == 0 || string(payload) == "null" {
return nil, nil
}
if getter == nil {
var res map[string]any
if err := json.Unmarshal(payload, &res); err != nil {
return nil, err
}
return res, nil
}
typed := getter()
if err := json.Unmarshal(payload, typed); err != nil {
return nil, err
}
return reflect.ValueOf(typed).Elem().Interface(), nil
}
func (s *metadataStore) get(name string) (Metadata, error) {
m, err := s.getByID(contextdirOf(name))
if err != nil {
return m, fmt.Errorf("context %q: %w", name, err)
}
return m, nil
}
func (s *metadataStore) getByID(id contextdir) (Metadata, error) {
fileName := filepath.Join(s.contextDir(id), metaFile)
bytes, err := os.ReadFile(fileName)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return Metadata{}, notFound(fmt.Errorf("context not found: %w", err))
}
return Metadata{}, err
}
var untyped untypedContextMetadata
r := Metadata{
Endpoints: make(map[string]any),
}
if err := json.Unmarshal(bytes, &untyped); err != nil {
return Metadata{}, fmt.Errorf("parsing %s: %v", fileName, err)
}
r.Name = untyped.Name
if r.Metadata, err = parseTypedOrMap(untyped.Metadata, s.config.contextType); err != nil {
return Metadata{}, fmt.Errorf("parsing %s: %v", fileName, err)
}
for k, v := range untyped.Endpoints {
if r.Endpoints[k], err = parseTypedOrMap(v, s.config.endpointTypes[k]); err != nil {
return Metadata{}, fmt.Errorf("parsing %s: %v", fileName, err)
}
}
return r, err
}
func (s *metadataStore) remove(name string) error {
if err := os.RemoveAll(s.contextDir(contextdirOf(name))); err != nil {
return fmt.Errorf("failed to remove metadata: %w", err)
}
return nil
}
func (s *metadataStore) list() ([]Metadata, error) {
ctxDirs, err := listRecursivelyMetadataDirs(s.root)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, err
}
res := make([]Metadata, 0, len(ctxDirs))
for _, dir := range ctxDirs {
c, err := s.getByID(contextdir(dir))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
return nil, fmt.Errorf("failed to read metadata: %w", err)
}
res = append(res, c)
}
sort.Slice(res, func(i, j int) bool {
return sortorder.NaturalLess(res[i].Name, res[j].Name)
})
return res, nil
}
func isContextDir(path string) bool {
s, err := os.Stat(filepath.Join(path, metaFile))
if err != nil {
return false
}
return !s.IsDir()
}
func listRecursivelyMetadataDirs(root string) ([]string, error) {
fis, err := os.ReadDir(root)
if err != nil {
return nil, err
}
var result []string
for _, fi := range fis {
if fi.IsDir() {
if isContextDir(filepath.Join(root, fi.Name())) {
result = append(result, fi.Name())
}
subs, err := listRecursivelyMetadataDirs(filepath.Join(root, fi.Name()))
if err != nil {
return nil, err
}
for _, s := range subs {
result = append(result, filepath.Join(fi.Name(), s))
}
}
}
return result, nil
}
type untypedContextMetadata struct {
Metadata json.RawMessage `json:"metadata,omitempty"`
Endpoints map[string]json.RawMessage `json:"endpoints,omitempty"`
Name string `json:"name,omitempty"`
}

View File

@@ -0,0 +1,507 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package store
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
_ "crypto/sha256" // ensure ids can be computed
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path"
"path/filepath"
"strings"
"github.com/docker/cli/internal/lazyregexp"
"github.com/opencontainers/go-digest"
)
const restrictedNamePattern = "^[a-zA-Z0-9][a-zA-Z0-9_.+-]+$"
var restrictedNameRegEx = lazyregexp.New(restrictedNamePattern)
// Store provides a context store for easily remembering endpoints configuration
type Store interface {
Reader
Lister
Writer
StorageInfoProvider
}
// Reader provides read-only (without list) access to context data
type Reader interface {
GetMetadata(name string) (Metadata, error)
ListTLSFiles(name string) (map[string]EndpointFiles, error)
GetTLSData(contextName, endpointName, fileName string) ([]byte, error)
}
// Lister provides listing of contexts
type Lister interface {
List() ([]Metadata, error)
}
// ReaderLister combines Reader and Lister interfaces
type ReaderLister interface {
Reader
Lister
}
// StorageInfoProvider provides more information about storage details of contexts
type StorageInfoProvider interface {
GetStorageInfo(contextName string) StorageInfo
}
// Writer provides write access to context data
type Writer interface {
CreateOrUpdate(meta Metadata) error
Remove(name string) error
ResetTLSMaterial(name string, data *ContextTLSData) error
ResetEndpointTLSMaterial(contextName string, endpointName string, data *EndpointTLSData) error
}
// ReaderWriter combines Reader and Writer interfaces
type ReaderWriter interface {
Reader
Writer
}
// Metadata contains metadata about a context and its endpoints
type Metadata struct {
Name string `json:",omitempty"`
Metadata any `json:",omitempty"`
Endpoints map[string]any `json:",omitempty"`
}
// StorageInfo contains data about where a given context is stored
type StorageInfo struct {
MetadataPath string
TLSPath string
}
// EndpointTLSData represents tls data for a given endpoint
type EndpointTLSData struct {
Files map[string][]byte
}
// ContextTLSData represents tls data for a whole context
type ContextTLSData struct {
Endpoints map[string]EndpointTLSData
}
// New creates a store from a given directory.
// If the directory does not exist or is empty, initialize it
func New(dir string, cfg Config) *ContextStore {
metaRoot := filepath.Join(dir, metadataDir)
tlsRoot := filepath.Join(dir, tlsDir)
return &ContextStore{
meta: &metadataStore{
root: metaRoot,
config: cfg,
},
tls: &tlsStore{
root: tlsRoot,
},
}
}
// ContextStore implements Store.
type ContextStore struct {
meta *metadataStore
tls *tlsStore
}
// List return all contexts.
func (s *ContextStore) List() ([]Metadata, error) {
return s.meta.list()
}
// Names return Metadata names for a Lister
func Names(s Lister) ([]string, error) {
if s == nil {
return nil, errors.New("nil lister")
}
list, err := s.List()
if err != nil {
return nil, err
}
names := make([]string, 0, len(list))
for _, item := range list {
names = append(names, item.Name)
}
return names, nil
}
// CreateOrUpdate creates or updates metadata for the context.
func (s *ContextStore) CreateOrUpdate(meta Metadata) error {
return s.meta.createOrUpdate(meta)
}
// Remove deletes the context with the given name, if found.
func (s *ContextStore) Remove(name string) error {
if err := s.meta.remove(name); err != nil {
return fmt.Errorf("failed to remove context %s: %w", name, err)
}
if err := s.tls.remove(name); err != nil {
return fmt.Errorf("failed to remove context %s: %w", name, err)
}
return nil
}
// GetMetadata returns the metadata for the context with the given name.
// It returns an errdefs.ErrNotFound if the context was not found.
func (s *ContextStore) GetMetadata(name string) (Metadata, error) {
return s.meta.get(name)
}
// ResetTLSMaterial removes TLS data for all endpoints in the context and replaces
// it with the new data.
func (s *ContextStore) ResetTLSMaterial(name string, data *ContextTLSData) error {
if err := s.tls.remove(name); err != nil {
return err
}
if data == nil {
return nil
}
for ep, files := range data.Endpoints {
for fileName, data := range files.Files {
if err := s.tls.createOrUpdate(name, ep, fileName, data); err != nil {
return err
}
}
}
return nil
}
// ResetEndpointTLSMaterial removes TLS data for the given context and endpoint,
// and replaces it with the new data.
func (s *ContextStore) ResetEndpointTLSMaterial(contextName string, endpointName string, data *EndpointTLSData) error {
if err := s.tls.removeEndpoint(contextName, endpointName); err != nil {
return err
}
if data == nil {
return nil
}
for fileName, data := range data.Files {
if err := s.tls.createOrUpdate(contextName, endpointName, fileName, data); err != nil {
return err
}
}
return nil
}
// ListTLSFiles returns the list of TLS files present for each endpoint in the
// context.
func (s *ContextStore) ListTLSFiles(name string) (map[string]EndpointFiles, error) {
return s.tls.listContextData(name)
}
// GetTLSData reads, and returns the content of the given fileName for an endpoint.
// It returns an errdefs.ErrNotFound if the file was not found.
func (s *ContextStore) GetTLSData(contextName, endpointName, fileName string) ([]byte, error) {
return s.tls.getData(contextName, endpointName, fileName)
}
// GetStorageInfo returns the paths where the Metadata and TLS data are stored
// for the context.
func (s *ContextStore) GetStorageInfo(contextName string) StorageInfo {
return StorageInfo{
MetadataPath: s.meta.contextDir(contextdirOf(contextName)),
TLSPath: s.tls.contextDir(contextName),
}
}
// ValidateContextName checks a context name is valid.
func ValidateContextName(name string) error {
if name == "" {
return errors.New("context name cannot be empty")
}
if name == "default" {
return errors.New(`"default" is a reserved context name`)
}
if !restrictedNameRegEx.MatchString(name) {
return fmt.Errorf("context name %q is invalid, names are validated against regexp %q", name, restrictedNamePattern)
}
return nil
}
// Export exports an existing namespace into an opaque data stream
// This stream is actually a tarball containing context metadata and TLS materials, but it does
// not map 1:1 the layout of the context store (don't try to restore it manually without calling store.Import)
func Export(name string, s Reader) io.ReadCloser {
reader, writer := io.Pipe()
go func() {
tw := tar.NewWriter(writer)
defer tw.Close()
defer writer.Close()
meta, err := s.GetMetadata(name)
if err != nil {
writer.CloseWithError(err)
return
}
metaBytes, err := json.Marshal(&meta)
if err != nil {
writer.CloseWithError(err)
return
}
if err = tw.WriteHeader(&tar.Header{
Name: metaFile,
Mode: 0o644,
Size: int64(len(metaBytes)),
}); err != nil {
writer.CloseWithError(err)
return
}
if _, err = tw.Write(metaBytes); err != nil {
writer.CloseWithError(err)
return
}
tlsFiles, err := s.ListTLSFiles(name)
if err != nil {
writer.CloseWithError(err)
return
}
if err = tw.WriteHeader(&tar.Header{
Name: "tls",
Mode: 0o700,
Size: 0,
Typeflag: tar.TypeDir,
}); err != nil {
writer.CloseWithError(err)
return
}
for endpointName, endpointFiles := range tlsFiles {
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("tls", endpointName),
Mode: 0o700,
Size: 0,
Typeflag: tar.TypeDir,
}); err != nil {
writer.CloseWithError(err)
return
}
for _, fileName := range endpointFiles {
data, err := s.GetTLSData(name, endpointName, fileName)
if err != nil {
writer.CloseWithError(err)
return
}
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("tls", endpointName, fileName),
Mode: 0o600,
Size: int64(len(data)),
}); err != nil {
writer.CloseWithError(err)
return
}
if _, err = tw.Write(data); err != nil {
writer.CloseWithError(err)
return
}
}
}
}()
return reader
}
const (
maxAllowedFileSizeToImport int64 = 10 << 20
zipType string = "application/zip"
)
func getImportContentType(r *bufio.Reader) (string, error) {
head, err := r.Peek(512)
if err != nil && err != io.EOF {
return "", err
}
return http.DetectContentType(head), nil
}
// Import imports an exported context into a store
func Import(name string, s Writer, reader io.Reader) error {
// Buffered reader will not advance the buffer, needed to determine content type
r := bufio.NewReader(reader)
importContentType, err := getImportContentType(r)
if err != nil {
return err
}
switch importContentType {
case zipType:
return importZip(name, s, r)
default:
// Assume it's a TAR (TAR does not have a "magic number")
return importTar(name, s, r)
}
}
func isValidFilePath(p string) error {
if p != metaFile && !strings.HasPrefix(p, "tls/") {
return errors.New("unexpected context file")
}
if path.Clean(p) != p {
return errors.New("unexpected path format")
}
if strings.Contains(p, `\`) {
return errors.New(`unexpected '\' in path`)
}
return nil
}
func importTar(name string, s Writer, reader io.Reader) error {
tr := tar.NewReader(&limitedReader{R: reader, N: maxAllowedFileSizeToImport})
tlsData := ContextTLSData{
Endpoints: map[string]EndpointTLSData{},
}
var importedMetaFile bool
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if hdr.Typeflag != tar.TypeReg {
// skip this entry, only taking files into account
continue
}
if err := isValidFilePath(hdr.Name); err != nil {
return fmt.Errorf("%s: %w", hdr.Name, err)
}
if hdr.Name == metaFile {
data, err := io.ReadAll(tr)
if err != nil {
return err
}
meta, err := parseMetadata(data, name)
if err != nil {
return err
}
if err := s.CreateOrUpdate(meta); err != nil {
return err
}
importedMetaFile = true
} else if strings.HasPrefix(hdr.Name, "tls/") {
data, err := io.ReadAll(tr)
if err != nil {
return err
}
if err := importEndpointTLS(&tlsData, hdr.Name, data); err != nil {
return err
}
}
}
if !importedMetaFile {
return invalidParameter(errors.New("invalid context: no metadata found"))
}
return s.ResetTLSMaterial(name, &tlsData)
}
func importZip(name string, s Writer, reader io.Reader) error {
body, err := io.ReadAll(&limitedReader{R: reader, N: maxAllowedFileSizeToImport})
if err != nil {
return err
}
zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
return err
}
tlsData := ContextTLSData{
Endpoints: map[string]EndpointTLSData{},
}
var importedMetaFile bool
for _, zf := range zr.File {
fi := zf.FileInfo()
if !fi.Mode().IsRegular() {
// skip this entry, only taking regular files into account
continue
}
if err := isValidFilePath(zf.Name); err != nil {
return fmt.Errorf("%s: %w", zf.Name, err)
}
if zf.Name == metaFile {
f, err := zf.Open()
if err != nil {
return err
}
data, err := io.ReadAll(&limitedReader{R: f, N: maxAllowedFileSizeToImport})
defer f.Close()
if err != nil {
return err
}
meta, err := parseMetadata(data, name)
if err != nil {
return err
}
if err := s.CreateOrUpdate(meta); err != nil {
return err
}
importedMetaFile = true
} else if strings.HasPrefix(zf.Name, "tls/") {
f, err := zf.Open()
if err != nil {
return err
}
data, err := io.ReadAll(f)
defer f.Close()
if err != nil {
return err
}
err = importEndpointTLS(&tlsData, zf.Name, data)
if err != nil {
return err
}
}
}
if !importedMetaFile {
return invalidParameter(errors.New("invalid context: no metadata found"))
}
return s.ResetTLSMaterial(name, &tlsData)
}
func parseMetadata(data []byte, name string) (Metadata, error) {
var meta Metadata
if err := json.Unmarshal(data, &meta); err != nil {
return meta, err
}
if err := ValidateContextName(name); err != nil {
return Metadata{}, err
}
meta.Name = name
return meta, nil
}
func importEndpointTLS(tlsData *ContextTLSData, tlsPath string, data []byte) error {
parts := strings.SplitN(strings.TrimPrefix(tlsPath, "tls/"), "/", 2)
if len(parts) != 2 {
// TLS endpoints require archived file directory with 2 layers
// i.e. tls/{endpointName}/{fileName}
return errors.New("archive format is invalid")
}
epName := parts[0]
fileName := parts[1]
if _, ok := tlsData.Endpoints[epName]; !ok {
tlsData.Endpoints[epName] = EndpointTLSData{
Files: map[string][]byte{},
}
}
tlsData.Endpoints[epName].Files[fileName] = data
return nil
}
type contextdir string
func contextdirOf(name string) contextdir {
return contextdir(digest.FromString(name).Encoded())
}

View File

@@ -0,0 +1,56 @@
// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
//go:build go1.23
package store
// TypeGetter is a func used to determine the concrete type of a context or
// endpoint metadata by returning a pointer to an instance of the object
// eg: for a context of type DockerContext, the corresponding TypeGetter should return new(DockerContext)
type TypeGetter func() any
// NamedTypeGetter is a TypeGetter associated with a name
type NamedTypeGetter struct {
name string
typeGetter TypeGetter
}
// EndpointTypeGetter returns a NamedTypeGetter with the specified name and getter
func EndpointTypeGetter(name string, getter TypeGetter) NamedTypeGetter {
return NamedTypeGetter{
name: name,
typeGetter: getter,
}
}
// Config is used to configure the metadata marshaler of the context ContextStore
type Config struct {
contextType TypeGetter
endpointTypes map[string]TypeGetter
}
// SetEndpoint set an endpoint typing information
func (c Config) SetEndpoint(name string, getter TypeGetter) {
c.endpointTypes[name] = getter
}
// ForeachEndpointType calls cb on every endpoint type registered with the Config
func (c Config) ForeachEndpointType(cb func(string, TypeGetter) error) error {
for n, ep := range c.endpointTypes {
if err := cb(n, ep); err != nil {
return err
}
}
return nil
}
// NewConfig creates a config object
func NewConfig(contextType TypeGetter, endpoints ...NamedTypeGetter) Config {
res := Config{
contextType: contextType,
endpointTypes: make(map[string]TypeGetter),
}
for _, e := range endpoints {
res.endpointTypes[e.name] = e.typeGetter
}
return res
}

View File

@@ -0,0 +1,95 @@
package store
import (
"fmt"
"os"
"path/filepath"
"github.com/moby/sys/atomicwriter"
)
const tlsDir = "tls"
type tlsStore struct {
root string
}
func (s *tlsStore) contextDir(name string) string {
return filepath.Join(s.root, string(contextdirOf(name)))
}
func (s *tlsStore) endpointDir(name, endpointName string) string {
return filepath.Join(s.contextDir(name), endpointName)
}
func (s *tlsStore) createOrUpdate(name, endpointName, filename string, data []byte) error {
parentOfRoot := filepath.Dir(s.root)
if err := os.MkdirAll(parentOfRoot, 0o755); err != nil {
return err
}
endpointDir := s.endpointDir(name, endpointName)
if err := os.MkdirAll(endpointDir, 0o700); err != nil {
return err
}
return atomicwriter.WriteFile(filepath.Join(endpointDir, filename), data, 0o600)
}
func (s *tlsStore) getData(name, endpointName, filename string) ([]byte, error) {
data, err := os.ReadFile(filepath.Join(s.endpointDir(name, endpointName), filename))
if err != nil {
if os.IsNotExist(err) {
return nil, notFound(fmt.Errorf("TLS data for %s/%s/%s does not exist", name, endpointName, filename))
}
return nil, fmt.Errorf("failed to read TLS data for endpoint %s: %w", endpointName, err)
}
return data, nil
}
// remove deletes all TLS data for the given context.
func (s *tlsStore) remove(name string) error {
if err := os.RemoveAll(s.contextDir(name)); err != nil {
return fmt.Errorf("failed to remove TLS data: %w", err)
}
return nil
}
func (s *tlsStore) removeEndpoint(name, endpointName string) error {
if err := os.RemoveAll(s.endpointDir(name, endpointName)); err != nil {
return fmt.Errorf("failed to remove TLS data for endpoint %s: %w", endpointName, err)
}
return nil
}
func (s *tlsStore) listContextData(name string) (map[string]EndpointFiles, error) {
contextDir := s.contextDir(name)
epFSs, err := os.ReadDir(contextDir)
if err != nil {
if os.IsNotExist(err) {
return map[string]EndpointFiles{}, nil
}
return nil, fmt.Errorf("failed to list TLS files for context %s: %w", name, err)
}
r := make(map[string]EndpointFiles)
for _, epFS := range epFSs {
if epFS.IsDir() {
fss, err := os.ReadDir(filepath.Join(contextDir, epFS.Name()))
if os.IsNotExist(err) {
continue
}
if err != nil {
return nil, fmt.Errorf("failed to list TLS files for endpoint %s: %w", epFS.Name(), err)
}
var files EndpointFiles
for _, fs := range fss {
if !fs.IsDir() {
files = append(files, fs.Name())
}
}
r[epFS.Name()] = files
}
}
return r, nil
}
// EndpointFiles is a slice of strings representing file names
type EndpointFiles []string

View File

@@ -0,0 +1,98 @@
package context
import (
"os"
"github.com/docker/cli/cli/context/store"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
caKey = "ca.pem"
certKey = "cert.pem"
keyKey = "key.pem"
)
// TLSData holds ca/cert/key raw data
type TLSData struct {
CA []byte
Key []byte
Cert []byte
}
// ToStoreTLSData converts TLSData to the store representation
func (data *TLSData) ToStoreTLSData() *store.EndpointTLSData {
if data == nil {
return nil
}
result := store.EndpointTLSData{
Files: make(map[string][]byte),
}
if data.CA != nil {
result.Files[caKey] = data.CA
}
if data.Cert != nil {
result.Files[certKey] = data.Cert
}
if data.Key != nil {
result.Files[keyKey] = data.Key
}
return &result
}
// LoadTLSData loads TLS data from the store
func LoadTLSData(s store.Reader, contextName, endpointName string) (*TLSData, error) {
tlsFiles, err := s.ListTLSFiles(contextName)
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve TLS files for context %q", contextName)
}
if epTLSFiles, ok := tlsFiles[endpointName]; ok {
var tlsData TLSData
for _, f := range epTLSFiles {
data, err := s.GetTLSData(contextName, endpointName, f)
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve TLS data (%s) for context %q", f, contextName)
}
switch f {
case caKey:
tlsData.CA = data
case certKey:
tlsData.Cert = data
case keyKey:
tlsData.Key = data
default:
logrus.Warnf("unknown file in context %s TLS bundle: %s", contextName, f)
}
}
return &tlsData, nil
}
return nil, nil
}
// TLSDataFromFiles reads files into a TLSData struct (or returns nil if all paths are empty)
func TLSDataFromFiles(caPath, certPath, keyPath string) (*TLSData, error) {
var (
ca, cert, key []byte
err error
)
if caPath != "" {
if ca, err = os.ReadFile(caPath); err != nil {
return nil, err
}
}
if certPath != "" {
if cert, err = os.ReadFile(certPath); err != nil {
return nil, err
}
}
if keyPath != "" {
if key, err = os.ReadFile(keyPath); err != nil {
return nil, err
}
}
if ca == nil && cert == nil && key == nil {
return nil, nil
}
return &TLSData{CA: ca, Cert: cert, Key: key}, nil
}

View File

@@ -0,0 +1,37 @@
package debug
import (
"os"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
)
// Enable sets the DEBUG env var to true
// and makes the logger to log at debug level.
func Enable() {
_ = os.Setenv("DEBUG", "1")
logrus.SetLevel(logrus.DebugLevel)
}
// Disable sets the DEBUG env var to false
// and makes the logger to log at info level.
func Disable() {
_ = os.Setenv("DEBUG", "")
logrus.SetLevel(logrus.InfoLevel)
}
// IsEnabled checks whether the debug flag is set or not.
func IsEnabled() bool {
return os.Getenv("DEBUG") != ""
}
// OTELErrorHandler is an error handler for OTEL that
// uses the CLI debug package to log messages when an error
// occurs.
//
// The default is to log to the debug level which is only
// enabled when debugging is enabled.
var OTELErrorHandler otel.ErrorHandler = otel.ErrorHandlerFunc(func(err error) {
logrus.WithError(err).Debug("otel error")
})

View File

@@ -0,0 +1,148 @@
package flags
import (
"fmt"
"os"
"path/filepath"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/opts"
"github.com/docker/docker/client"
"github.com/docker/go-connections/tlsconfig"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
)
const (
// EnvEnableTLS is the name of the environment variable that can be used
// to enable TLS for client connections. When set to a non-empty value, TLS
// is enabled for API connections using TCP. For backward-compatibility, this
// environment-variable can only be used to enable TLS, not to disable.
//
// Note that TLS is always enabled implicitly if the "--tls-verify" option
// or "DOCKER_TLS_VERIFY" ([github.com/docker/docker/client.EnvTLSVerify])
// env var is set to, which could be to either enable or disable TLS certification
// validation. In both cases, TLS is enabled but, depending on the setting,
// with verification disabled.
EnvEnableTLS = "DOCKER_TLS"
// DefaultCaFile is the default filename for the CA pem file
DefaultCaFile = "ca.pem"
// DefaultKeyFile is the default filename for the key pem file
DefaultKeyFile = "key.pem"
// DefaultCertFile is the default filename for the cert pem file
DefaultCertFile = "cert.pem"
// FlagTLSVerify is the flag name for the TLS verification option
FlagTLSVerify = "tlsverify"
// FormatHelp describes the --format flag behavior for list commands
FormatHelp = `Format output using a custom template:
'table': Print output in table format with column headers (default)
'table TEMPLATE': Print output in table format using the given Go template
'json': Print in JSON format
'TEMPLATE': Print output using the given Go template.
Refer to https://docs.docker.com/go/formatting/ for more information about formatting output with templates`
// InspectFormatHelp describes the --format flag behavior for inspect commands
InspectFormatHelp = `Format output using a custom template:
'json': Print in JSON format
'TEMPLATE': Print output using the given Go template.
Refer to https://docs.docker.com/go/formatting/ for more information about formatting output with templates`
)
var (
dockerCertPath = os.Getenv(client.EnvOverrideCertPath)
dockerTLSVerify = os.Getenv(client.EnvTLSVerify) != ""
dockerTLS = os.Getenv(EnvEnableTLS) != ""
)
// ClientOptions are the options used to configure the client cli.
type ClientOptions struct {
Debug bool
Hosts []string
LogLevel string
TLS bool
TLSVerify bool
TLSOptions *tlsconfig.Options
Context string
ConfigDir string
}
// NewClientOptions returns a new ClientOptions.
func NewClientOptions() *ClientOptions {
return &ClientOptions{}
}
// InstallFlags adds flags for the common options on the FlagSet
func (o *ClientOptions) InstallFlags(flags *pflag.FlagSet) {
configDir := config.Dir()
if dockerCertPath == "" {
dockerCertPath = configDir
}
flags.StringVar(&o.ConfigDir, "config", configDir, "Location of client config files")
flags.BoolVarP(&o.Debug, "debug", "D", false, "Enable debug mode")
flags.StringVarP(&o.LogLevel, "log-level", "l", "info", `Set the logging level ("debug", "info", "warn", "error", "fatal")`)
flags.BoolVar(&o.TLS, "tls", dockerTLS, "Use TLS; implied by --tlsverify")
flags.BoolVar(&o.TLSVerify, FlagTLSVerify, dockerTLSVerify, "Use TLS and verify the remote")
o.TLSOptions = &tlsconfig.Options{
CAFile: filepath.Join(dockerCertPath, DefaultCaFile),
CertFile: filepath.Join(dockerCertPath, DefaultCertFile),
KeyFile: filepath.Join(dockerCertPath, DefaultKeyFile),
}
tlsOptions := o.TLSOptions
flags.Var(opts.NewQuotedString(&tlsOptions.CAFile), "tlscacert", "Trust certs signed only by this CA")
flags.Var(opts.NewQuotedString(&tlsOptions.CertFile), "tlscert", "Path to TLS certificate file")
flags.Var(opts.NewQuotedString(&tlsOptions.KeyFile), "tlskey", "Path to TLS key file")
// opts.ValidateHost is not used here, so as to allow connection helpers
hostOpt := opts.NewNamedListOptsRef("hosts", &o.Hosts, nil)
flags.VarP(hostOpt, "host", "H", "Daemon socket to connect to")
flags.StringVarP(&o.Context, "context", "c", "",
`Name of the context to use to connect to the daemon (overrides `+client.EnvOverrideHost+` env var and default context set with "docker context use")`)
}
// SetDefaultOptions sets default values for options after flag parsing is
// complete
func (o *ClientOptions) SetDefaultOptions(flags *pflag.FlagSet) {
// Regardless of whether the user sets it to true or false, if they
// specify --tlsverify at all then we need to turn on TLS
// TLSVerify can be true even if not set due to DOCKER_TLS_VERIFY env var, so we need
// to check that here as well
if flags.Changed(FlagTLSVerify) || o.TLSVerify {
o.TLS = true
}
if !o.TLS {
o.TLSOptions = nil
} else {
tlsOptions := o.TLSOptions
tlsOptions.InsecureSkipVerify = !o.TLSVerify
// Reset CertFile and KeyFile to empty string if the user did not specify
// the respective flags and the respective default files were not found.
if !flags.Changed("tlscert") {
if _, err := os.Stat(tlsOptions.CertFile); os.IsNotExist(err) {
tlsOptions.CertFile = ""
}
}
if !flags.Changed("tlskey") {
if _, err := os.Stat(tlsOptions.KeyFile); os.IsNotExist(err) {
tlsOptions.KeyFile = ""
}
}
}
}
// SetLogLevel sets the logrus logging level
func SetLogLevel(logLevel string) {
if logLevel != "" {
lvl, err := logrus.ParseLevel(logLevel)
if err != nil {
_, _ = fmt.Fprintln(os.Stderr, "Unable to parse logging level:", logLevel)
os.Exit(1)
}
logrus.SetLevel(lvl)
} else {
logrus.SetLevel(logrus.InfoLevel)
}
}

View File

@@ -0,0 +1,20 @@
package hints
import (
"os"
"strconv"
)
// Enabled returns whether cli hints are enabled or not. Hints are enabled by
// default, but can be disabled through the "DOCKER_CLI_HINTS" environment
// variable.
func Enabled() bool {
if v := os.Getenv("DOCKER_CLI_HINTS"); v != "" {
enabled, err := strconv.ParseBool(v)
if err != nil {
return true
}
return enabled
}
return true
}

View File

@@ -0,0 +1,61 @@
package streams
import (
"errors"
"io"
"os"
"runtime"
"github.com/moby/term"
)
// In is an input stream to read user input. It implements [io.ReadCloser]
// with additional utilities, such as putting the terminal in raw mode.
type In struct {
commonStream
in io.ReadCloser
}
// Read implements the [io.Reader] interface.
func (i *In) Read(p []byte) (int, error) {
return i.in.Read(p)
}
// Close implements the [io.Closer] interface.
func (i *In) Close() error {
return i.in.Close()
}
// SetRawTerminal sets raw mode on the input terminal. It is a no-op if In
// is not a TTY, or if the "NORAW" environment variable is set to a non-empty
// value.
func (i *In) SetRawTerminal() (err error) {
if !i.isTerminal || os.Getenv("NORAW") != "" {
return nil
}
i.state, err = term.SetRawTerminal(i.fd)
return err
}
// CheckTty checks if we are trying to attach to a container TTY
// from a non-TTY client input stream, and if so, returns an error.
func (i *In) CheckTty(attachStdin, ttyMode bool) error {
// In order to attach to a container tty, input stream for the client must
// be a tty itself: redirecting or piping the client standard input is
// incompatible with `docker run -t`, `docker exec -t` or `docker attach`.
if ttyMode && attachStdin && !i.isTerminal {
const eText = "the input device is not a TTY"
if runtime.GOOS == "windows" {
return errors.New(eText + ". If you are using mintty, try prefixing the command with 'winpty'")
}
return errors.New(eText)
}
return nil
}
// NewIn returns a new [In] from an [io.ReadCloser].
func NewIn(in io.ReadCloser) *In {
i := &In{in: in}
i.fd, i.isTerminal = term.GetFdInfo(in)
return i
}

View File

@@ -0,0 +1,58 @@
package streams
import (
"io"
"os"
"github.com/moby/term"
"github.com/sirupsen/logrus"
)
// Out is an output stream to write normal program output. It implements
// an [io.Writer], with additional utilities for detecting whether a terminal
// is connected, getting the TTY size, and putting the terminal in raw mode.
type Out struct {
commonStream
out io.Writer
}
func (o *Out) Write(p []byte) (int, error) {
return o.out.Write(p)
}
// SetRawTerminal puts the output of the terminal connected to the stream
// into raw mode.
//
// On UNIX, this does nothing. On Windows, it disables LF -> CRLF/ translation.
// It is a no-op if Out is not a TTY, or if the "NORAW" environment variable is
// set to a non-empty value.
func (o *Out) SetRawTerminal() (err error) {
if !o.isTerminal || os.Getenv("NORAW") != "" {
return nil
}
o.state, err = term.SetRawTerminalOutput(o.fd)
return err
}
// GetTtySize returns the height and width in characters of the TTY, or
// zero for both if no TTY is connected.
func (o *Out) GetTtySize() (height uint, width uint) {
if !o.isTerminal {
return 0, 0
}
ws, err := term.GetWinsize(o.fd)
if err != nil {
logrus.WithError(err).Debug("Error getting TTY size")
if ws == nil {
return 0, 0
}
}
return uint(ws.Height), uint(ws.Width)
}
// NewOut returns a new [Out] from an [io.Writer].
func NewOut(out io.Writer) *Out {
o := &Out{out: out}
o.fd, o.isTerminal = term.GetFdInfo(out)
return o
}

Some files were not shown because too many files have changed in this diff Show More