Merge pull request #3648 from dave-tucker/skip-unsup

pkg_build: Allow skipping unsupported platforms

docs/packages.md

@@ -105,6 +105,15 @@ As of this writing, those are:
 * `linux/arm64`
 * `linux/s390x`
 
+You can choose to skip one of the platforms from `build.yml` or those selected
+by default using the `--skip-platforms` flag.
+
+For example:
+
+```
+linuxkit pkg build --skip-platforms linux/s390x ...
+```
+
 You can override the target build platform by passing it the `--platforms` option:
 
 ```

src/cmd/linuxkit/pkg_build.go

@@ -28,6 +28,7 @@ func pkgBuild(args []string) {
 	force := flags.Bool("force", false, "Force rebuild")
 	docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
 	platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
+	skipPlatforms := flags.String("skip-platforms", "", "Platforms that should be skipped, even if present in build.yml")
 	builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
 	buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
 
@@ -47,13 +48,34 @@ func pkgBuild(args []string) {
 	if *docker {
 		opts = append(opts, pkglib.WithBuildTargetDockerCache())
 	}
+
+	// skipPlatformsMap contains platforms that should be skipped
+	skipPlatformsMap := make(map[string]bool)
+	if *skipPlatforms != "" {
+		for _, platform := range strings.Split(*skipPlatforms, ",") {
+			parts := strings.SplitN(platform, "/", 2)
+			if len(parts) != 2 || parts[0] == "" || parts[0] != "linux" || parts[1] == "" {
+				fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", platform)
+				os.Exit(1)
+			}
+			skipPlatformsMap[strings.Trim(parts[1], " ")] = true
+		}
+	}
 	// if platforms requested is blank, use all from the config
 	var plats []imagespec.Platform
 	if *platforms == "" {
 		for _, a := range p.Arches() {
+			if _, ok := skipPlatformsMap[a]; ok {
+				continue
+			}
 			plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
 		}
 	} else {
+		// don't allow the use of --skip-platforms with --platforms
+		if *skipPlatforms != "" {
+			fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together")
+			os.Exit(1)
+		}
 		for _, p := range strings.Split(*platforms, ",") {
 			parts := strings.SplitN(p, "/", 2)
 			if len(parts) != 2 || parts[0] == "" || parts[1] == "" {

src/cmd/linuxkit/pkglib/build.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 
 	"github.com/containerd/containerd/reference"
-	"github.com/google/go-containerregistry/pkg/v1"
+	registry "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
 	lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
@@ -168,12 +168,6 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 		return fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err)
 	}
 
-	for _, platform := range bo.platforms {
-		if !p.archSupported(platform.Architecture) {
-			return fmt.Errorf("arch %s not supported by this package, skipping build", platform.Architecture)
-		}
-	}
-
 	if err := p.cleanForBuild(); err != nil {
 		return err
 	}
@@ -241,7 +235,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 		fmt.Fprintf(writer, "building %s\n", ref)
 		var (
 			args  []string
-			descs []v1.Descriptor
+			descs []registry.Descriptor
 		)
 
 		if p.git != nil && p.gitRepo != "" {
@@ -360,9 +354,9 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 }
 
 // buildArch builds the package for a single arch
-func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*v1.Descriptor, error) {
+func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*registry.Descriptor, error) {
 	var (
-		desc    *v1.Descriptor
+		desc    *registry.Descriptor
 		tagArch string
 		tag     = p.Tag()
 	)

src/cmd/linuxkit/pkglib/build_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 
 	"github.com/containerd/containerd/reference"
-	"github.com/google/go-containerregistry/pkg/v1"
+	registry "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 	lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -97,7 +97,7 @@ type cacheMocker struct {
 	enableImagePull        bool
 	enableImageLoad        bool
 	enableIndexWrite       bool
-	images                 map[string][]v1.Descriptor
+	images                 map[string][]registry.Descriptor
 	hashes                 map[string][]byte
 }
 
@@ -126,15 +126,15 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
 	if err != nil {
 		return nil, fmt.Errorf("error reading data: %v", err)
 	}
-	hash, size, err := v1.SHA256(bytes.NewReader(b))
+	hash, size, err := registry.SHA256(bytes.NewReader(b))
 	if err != nil {
 		return nil, fmt.Errorf("error calculating hash of layer: %v", err)
 	}
 	c.assignHash(hash.String(), b)
 
-	im := v1.Manifest{
+	im := registry.Manifest{
 		MediaType: types.OCIManifestSchema1,
-		Layers: []v1.Descriptor{
+		Layers: []registry.Descriptor{
 			{MediaType: types.OCILayer, Size: size, Digest: hash},
 		},
 		SchemaVersion: 2,
@@ -145,12 +145,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal new image to json: %v", err)
 	}
-	hash, size, err = v1.SHA256(bytes.NewReader(b))
+	hash, size, err = registry.SHA256(bytes.NewReader(b))
 	if err != nil {
 		return nil, fmt.Errorf("error calculating hash of index json: %v", err)
 	}
 	c.assignHash(hash.String(), b)
-	desc := v1.Descriptor{
+	desc := registry.Descriptor{
 		MediaType: types.OCIManifestSchema1,
 		Size:      size,
 		Digest:    hash,
@@ -163,12 +163,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
 	return c.NewSource(ref, "", &desc), nil
 }
 
-func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) {
+func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) {
 	if !c.enableIndexWrite {
 		return nil, errors.New("disabled")
 	}
 	image := ref.String()
-	im := v1.IndexManifest{
+	im := registry.IndexManifest{
 		MediaType:     types.OCIImageIndex,
 		Manifests:     descriptors,
 		SchemaVersion: 2,
@@ -179,12 +179,12 @@ func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descript
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
 	}
-	hash, size, err := v1.SHA256(bytes.NewReader(b))
+	hash, size, err := registry.SHA256(bytes.NewReader(b))
 	if err != nil {
 		return nil, fmt.Errorf("error calculating hash of index json: %v", err)
 	}
 	c.assignHash(hash.String(), b)
-	desc := v1.Descriptor{
+	desc := registry.Descriptor{
 		MediaType: types.OCIImageIndex,
 		Size:      size,
 		Digest:    hash,
@@ -206,13 +206,13 @@ func (c *cacheMocker) Push(name string) error {
 	return nil
 }
 
-func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) {
+func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) {
 	if !c.enabledDescriptorWrite {
 		return nil, errors.New("descriptor disabled")
 	}
 	var (
 		image = ref.String()
-		im    = v1.IndexManifest{
+		im    = registry.IndexManifest{
 			MediaType:     types.OCIImageIndex,
 			Manifests:     descriptors,
 			SchemaVersion: 2,
@@ -223,12 +223,12 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
 	}
-	hash, size, err := v1.SHA256(bytes.NewReader(b))
+	hash, size, err := registry.SHA256(bytes.NewReader(b))
 	if err != nil {
 		return nil, fmt.Errorf("error calculating hash of index json: %v", err)
 	}
 	c.assignHash(hash.String(), b)
-	root := v1.Descriptor{
+	root := registry.Descriptor{
 		MediaType: types.OCIImageIndex,
 		Size:      size,
 		Digest:    hash,
@@ -240,13 +240,13 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des
 
 	return c.NewSource(ref, "", &root), nil
 }
-func (c *cacheMocker) FindDescriptor(name string) (*v1.Descriptor, error) {
+func (c *cacheMocker) FindDescriptor(name string) (*registry.Descriptor, error) {
 	if desc, ok := c.images[name]; ok && len(desc) > 0 {
 		return &desc[0], nil
 	}
 	return nil, fmt.Errorf("not found %s", name)
 }
-func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *v1.Descriptor) lktspec.ImageSource {
+func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *registry.Descriptor) lktspec.ImageSource {
 	return cacheMockerSource{c, ref, architecture, descriptor}
 }
 func (c *cacheMocker) assignHash(hash string, b []byte) {
@@ -255,9 +255,9 @@ func (c *cacheMocker) assignHash(hash string, b []byte) {
 	}
 	c.hashes[hash] = b
 }
-func (c *cacheMocker) appendImage(image string, root v1.Descriptor) {
+func (c *cacheMocker) appendImage(image string, root registry.Descriptor) {
 	if c.images == nil {
-		c.images = map[string][]v1.Descriptor{}
+		c.images = map[string][]registry.Descriptor{}
 	}
 	c.images[image] = append(c.images[image], root)
 }
@@ -266,7 +266,7 @@ type cacheMockerSource struct {
 	c            *cacheMocker
 	ref          *reference.Spec
 	architecture string
-	descriptor   *v1.Descriptor
+	descriptor   *registry.Descriptor
 }
 
 func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) {
@@ -275,7 +275,7 @@ func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) {
 func (c cacheMockerSource) TarReader() (io.ReadCloser, error) {
 	return nil, errors.New("unsupported")
 }
-func (c cacheMockerSource) Descriptor() *v1.Descriptor {
+func (c cacheMockerSource) Descriptor() *registry.Descriptor {
 	return c.descriptor
 }
 
@@ -299,7 +299,6 @@ func TestBuild(t *testing.T) {
 		err     string
 	}{
 		{"invalid tag", Pkg{image: "docker.io/foo/bar:abc:def:ghi"}, nil, nil, &dockerMocker{}, &cacheMocker{}, "could not resolve references"},
-		{"mismatched platforms", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"arm64"}}, nil, []string{"amd64"}, nil, nil, fmt.Sprintf("arch %s not supported", "amd64")},
 		{"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"},
 		{"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must provide linuxkit build cache"},
 		{"unsupported buildkit", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "buildkit not supported, check docker version"},

src/cmd/linuxkit/pkglib/pkglib.go

@@ -277,6 +277,7 @@ func (p Pkg) Tag() string {
 	return p.org + "/" + p.image + ":" + t
 }
 
+// FullTag returns a reference expanded tag
 func (p Pkg) FullTag() string {
 	return util.ReferenceExpand(p.Tag())
 }