From 907dc5394ba173ce22bbf3d8cd05746a66474c93 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 2 May 2021 12:28:43 +0100 Subject: [PATCH 1/5] pkglib: Don't error on platform mismatch This prevents override of the platform by the user. lkt pkg build --platform=linux/amd64 pkg/bpftrace should attempt to build that package for that arch even though it is not in the build.yml Signed-off-by: Dave Tucker --- src/cmd/linuxkit/pkglib/build.go | 8 +------- src/cmd/linuxkit/pkglib/build_test.go | 3 +-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index d17333d07..041e88f55 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/containerd/containerd/reference" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" @@ -168,12 +168,6 @@ func (p Pkg) Build(bos ...BuildOpt) error { return fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err) } - for _, platform := range bo.platforms { - if !p.archSupported(platform.Architecture) { - return fmt.Errorf("arch %s not supported by this package, skipping build", platform.Architecture) - } - } - if err := p.cleanForBuild(); err != nil { return err } diff --git a/src/cmd/linuxkit/pkglib/build_test.go b/src/cmd/linuxkit/pkglib/build_test.go index 59efda9f1..7144c945e 100644 --- a/src/cmd/linuxkit/pkglib/build_test.go +++ b/src/cmd/linuxkit/pkglib/build_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/containerd/containerd/reference" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -299,7 +299,6 @@ func TestBuild(t *testing.T) { err string }{ {"invalid tag", Pkg{image: "docker.io/foo/bar:abc:def:ghi"}, nil, nil, &dockerMocker{}, &cacheMocker{}, "could not resolve references"}, - {"mismatched platforms", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"arm64"}}, nil, []string{"amd64"}, nil, nil, fmt.Sprintf("arch %s not supported", "amd64")}, {"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"}, {"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must provide linuxkit build cache"}, {"unsupported buildkit", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "buildkit not supported, check docker version"}, From 981fb7ff326237ed920d1330aaad192d5d284933 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 2 May 2021 12:34:49 +0100 Subject: [PATCH 2/5] pkglib: Add missing comment to FullTag Signed-off-by: Dave Tucker --- src/cmd/linuxkit/pkglib/pkglib.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 69a29f145..7534ed4e7 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -277,6 +277,7 @@ func (p Pkg) Tag() string { return p.org + "/" + p.image + ":" + t } +// FullTag returns a reference expanded tag func (p Pkg) FullTag() string { return util.ReferenceExpand(p.Tag()) } From 54185c8a3f1658ed9fbd597dd6fa0275b25cd9af Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 2 May 2021 12:36:24 +0100 Subject: [PATCH 3/5] pkglib: Rename v1 to registry Signed-off-by: Dave Tucker --- src/cmd/linuxkit/pkglib/build.go | 8 ++--- src/cmd/linuxkit/pkglib/build_test.go | 42 +++++++++++++-------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index 041e88f55..f5da82180 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/containerd/containerd/reference" - v1 "github.com/google/go-containerregistry/pkg/v1" + registry "github.com/google/go-containerregistry/pkg/v1" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" @@ -235,7 +235,7 @@ func (p Pkg) Build(bos ...BuildOpt) error { fmt.Fprintf(writer, "building %s\n", ref) var ( args []string - descs []v1.Descriptor + descs []registry.Descriptor ) if p.git != nil && p.gitRepo != "" { @@ -354,9 +354,9 @@ func (p Pkg) Build(bos ...BuildOpt) error { } // buildArch builds the package for a single arch -func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*v1.Descriptor, error) { +func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*registry.Descriptor, error) { var ( - desc *v1.Descriptor + desc *registry.Descriptor tagArch string tag = p.Tag() ) diff --git a/src/cmd/linuxkit/pkglib/build_test.go b/src/cmd/linuxkit/pkglib/build_test.go index 7144c945e..8d83ea461 100644 --- a/src/cmd/linuxkit/pkglib/build_test.go +++ b/src/cmd/linuxkit/pkglib/build_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/containerd/containerd/reference" - v1 "github.com/google/go-containerregistry/pkg/v1" + registry "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -97,7 +97,7 @@ type cacheMocker struct { enableImagePull bool enableImageLoad bool enableIndexWrite bool - images map[string][]v1.Descriptor + images map[string][]registry.Descriptor hashes map[string][]byte } @@ -126,15 +126,15 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string, if err != nil { return nil, fmt.Errorf("error reading data: %v", err) } - hash, size, err := v1.SHA256(bytes.NewReader(b)) + hash, size, err := registry.SHA256(bytes.NewReader(b)) if err != nil { return nil, fmt.Errorf("error calculating hash of layer: %v", err) } c.assignHash(hash.String(), b) - im := v1.Manifest{ + im := registry.Manifest{ MediaType: types.OCIManifestSchema1, - Layers: []v1.Descriptor{ + Layers: []registry.Descriptor{ {MediaType: types.OCILayer, Size: size, Digest: hash}, }, SchemaVersion: 2, @@ -145,12 +145,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string, if err != nil { return nil, fmt.Errorf("unable to marshal new image to json: %v", err) } - hash, size, err = v1.SHA256(bytes.NewReader(b)) + hash, size, err = registry.SHA256(bytes.NewReader(b)) if err != nil { return nil, fmt.Errorf("error calculating hash of index json: %v", err) } c.assignHash(hash.String(), b) - desc := v1.Descriptor{ + desc := registry.Descriptor{ MediaType: types.OCIManifestSchema1, Size: size, Digest: hash, @@ -163,12 +163,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string, return c.NewSource(ref, "", &desc), nil } -func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) { +func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) { if !c.enableIndexWrite { return nil, errors.New("disabled") } image := ref.String() - im := v1.IndexManifest{ + im := registry.IndexManifest{ MediaType: types.OCIImageIndex, Manifests: descriptors, SchemaVersion: 2, @@ -179,12 +179,12 @@ func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descript if err != nil { return nil, fmt.Errorf("unable to marshal new index to json: %v", err) } - hash, size, err := v1.SHA256(bytes.NewReader(b)) + hash, size, err := registry.SHA256(bytes.NewReader(b)) if err != nil { return nil, fmt.Errorf("error calculating hash of index json: %v", err) } c.assignHash(hash.String(), b) - desc := v1.Descriptor{ + desc := registry.Descriptor{ MediaType: types.OCIImageIndex, Size: size, Digest: hash, @@ -206,13 +206,13 @@ func (c *cacheMocker) Push(name string) error { return nil } -func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) { +func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) { if !c.enabledDescriptorWrite { return nil, errors.New("descriptor disabled") } var ( image = ref.String() - im = v1.IndexManifest{ + im = registry.IndexManifest{ MediaType: types.OCIImageIndex, Manifests: descriptors, SchemaVersion: 2, @@ -223,12 +223,12 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des if err != nil { return nil, fmt.Errorf("unable to marshal new index to json: %v", err) } - hash, size, err := v1.SHA256(bytes.NewReader(b)) + hash, size, err := registry.SHA256(bytes.NewReader(b)) if err != nil { return nil, fmt.Errorf("error calculating hash of index json: %v", err) } c.assignHash(hash.String(), b) - root := v1.Descriptor{ + root := registry.Descriptor{ MediaType: types.OCIImageIndex, Size: size, Digest: hash, @@ -240,13 +240,13 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des return c.NewSource(ref, "", &root), nil } -func (c *cacheMocker) FindDescriptor(name string) (*v1.Descriptor, error) { +func (c *cacheMocker) FindDescriptor(name string) (*registry.Descriptor, error) { if desc, ok := c.images[name]; ok && len(desc) > 0 { return &desc[0], nil } return nil, fmt.Errorf("not found %s", name) } -func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *v1.Descriptor) lktspec.ImageSource { +func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *registry.Descriptor) lktspec.ImageSource { return cacheMockerSource{c, ref, architecture, descriptor} } func (c *cacheMocker) assignHash(hash string, b []byte) { @@ -255,9 +255,9 @@ func (c *cacheMocker) assignHash(hash string, b []byte) { } c.hashes[hash] = b } -func (c *cacheMocker) appendImage(image string, root v1.Descriptor) { +func (c *cacheMocker) appendImage(image string, root registry.Descriptor) { if c.images == nil { - c.images = map[string][]v1.Descriptor{} + c.images = map[string][]registry.Descriptor{} } c.images[image] = append(c.images[image], root) } @@ -266,7 +266,7 @@ type cacheMockerSource struct { c *cacheMocker ref *reference.Spec architecture string - descriptor *v1.Descriptor + descriptor *registry.Descriptor } func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) { @@ -275,7 +275,7 @@ func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) { func (c cacheMockerSource) TarReader() (io.ReadCloser, error) { return nil, errors.New("unsupported") } -func (c cacheMockerSource) Descriptor() *v1.Descriptor { +func (c cacheMockerSource) Descriptor() *registry.Descriptor { return c.descriptor } From 0320678133df8ff4392751a9d850772be4039d38 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 2 May 2021 12:46:21 +0100 Subject: [PATCH 4/5] build: Add --skip-platforms flag This adds a --skip-platforms flag that can be used with lkt pkg build to ignore any arch specified in build.yml Signed-off-by: Dave Tucker --- src/cmd/linuxkit/pkg_build.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/cmd/linuxkit/pkg_build.go b/src/cmd/linuxkit/pkg_build.go index f9a75d7ce..86bac742b 100644 --- a/src/cmd/linuxkit/pkg_build.go +++ b/src/cmd/linuxkit/pkg_build.go @@ -28,6 +28,7 @@ func pkgBuild(args []string) { force := flags.Bool("force", false, "Force rebuild") docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache") platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built") + skipPlatforms := flags.String("skip-platforms", "", "Platforms that should be skipped, even if present in build.yml") builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes") buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker") @@ -47,13 +48,34 @@ func pkgBuild(args []string) { if *docker { opts = append(opts, pkglib.WithBuildTargetDockerCache()) } + + // skipPlatformsMap contains platforms that should be skipped + skipPlatformsMap := make(map[string]bool) + if *skipPlatforms != "" { + for _, platform := range strings.Split(*skipPlatforms, ",") { + parts := strings.SplitN(platform, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[0] != "linux" || parts[1] == "" { + fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", platform) + os.Exit(1) + } + skipPlatformsMap[strings.Trim(parts[1], " ")] = true + } + } // if platforms requested is blank, use all from the config var plats []imagespec.Platform if *platforms == "" { for _, a := range p.Arches() { + if _, ok := skipPlatformsMap[a]; ok { + continue + } plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a}) } } else { + // don't allow the use of --skip-platforms with --platforms + if *skipPlatforms != "" { + fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together") + os.Exit(1) + } for _, p := range strings.Split(*platforms, ",") { parts := strings.SplitN(p, "/", 2) if len(parts) != 2 || parts[0] == "" || parts[1] == "" { From 28f43e81218bd16b67411212c186457beed58b5e Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 2 May 2021 13:03:52 +0100 Subject: [PATCH 5/5] docs: Document --skip-arches Signed-off-by: Dave Tucker --- docs/packages.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/packages.md b/docs/packages.md index ab68510df..aa42b4211 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -105,6 +105,15 @@ As of this writing, those are: * `linux/arm64` * `linux/s390x` +You can choose to skip one of the platforms from `build.yml` or those selected +by default using the `--skip-platforms` flag. + +For example: + +``` +linuxkit pkg build --skip-platforms linux/s390x ... +``` + You can override the target build platform by passing it the `--platforms` option: ```