Merge pull request #3648 from dave-tucker/skip-unsup

pkg_build: Allow skipping unsupported platforms
This commit is contained in:
Rolf Neugebauer 2021-05-02 20:19:23 +01:00 committed by GitHub
commit f3282724f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 57 additions and 32 deletions

View File

@ -105,6 +105,15 @@ As of this writing, those are:
* `linux/arm64` * `linux/arm64`
* `linux/s390x` * `linux/s390x`
You can choose to skip one of the platforms from `build.yml` or those selected
by default using the `--skip-platforms` flag.
For example:
```
linuxkit pkg build --skip-platforms linux/s390x ...
```
You can override the target build platform by passing it the `--platforms` option: You can override the target build platform by passing it the `--platforms` option:
``` ```

View File

@ -28,6 +28,7 @@ func pkgBuild(args []string) {
force := flags.Bool("force", false, "Force rebuild") force := flags.Bool("force", false, "Force rebuild")
docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache") docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built") platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
skipPlatforms := flags.String("skip-platforms", "", "Platforms that should be skipped, even if present in build.yml")
builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes") builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker") buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
@ -47,13 +48,34 @@ func pkgBuild(args []string) {
if *docker { if *docker {
opts = append(opts, pkglib.WithBuildTargetDockerCache()) opts = append(opts, pkglib.WithBuildTargetDockerCache())
} }
// skipPlatformsMap contains platforms that should be skipped
skipPlatformsMap := make(map[string]bool)
if *skipPlatforms != "" {
for _, platform := range strings.Split(*skipPlatforms, ",") {
parts := strings.SplitN(platform, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[0] != "linux" || parts[1] == "" {
fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", platform)
os.Exit(1)
}
skipPlatformsMap[strings.Trim(parts[1], " ")] = true
}
}
// if platforms requested is blank, use all from the config // if platforms requested is blank, use all from the config
var plats []imagespec.Platform var plats []imagespec.Platform
if *platforms == "" { if *platforms == "" {
for _, a := range p.Arches() { for _, a := range p.Arches() {
if _, ok := skipPlatformsMap[a]; ok {
continue
}
plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a}) plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
} }
} else { } else {
// don't allow the use of --skip-platforms with --platforms
if *skipPlatforms != "" {
fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together")
os.Exit(1)
}
for _, p := range strings.Split(*platforms, ",") { for _, p := range strings.Split(*platforms, ",") {
parts := strings.SplitN(p, "/", 2) parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" { if len(parts) != 2 || parts[0] == "" || parts[1] == "" {

View File

@ -12,7 +12,7 @@ import (
"strings" "strings"
"github.com/containerd/containerd/reference" "github.com/containerd/containerd/reference"
"github.com/google/go-containerregistry/pkg/v1" registry "github.com/google/go-containerregistry/pkg/v1"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache"
lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
@ -168,12 +168,6 @@ func (p Pkg) Build(bos ...BuildOpt) error {
return fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err) return fmt.Errorf("could not resolve references for image %s: %v", p.Tag(), err)
} }
for _, platform := range bo.platforms {
if !p.archSupported(platform.Architecture) {
return fmt.Errorf("arch %s not supported by this package, skipping build", platform.Architecture)
}
}
if err := p.cleanForBuild(); err != nil { if err := p.cleanForBuild(); err != nil {
return err return err
} }
@ -241,7 +235,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
fmt.Fprintf(writer, "building %s\n", ref) fmt.Fprintf(writer, "building %s\n", ref)
var ( var (
args []string args []string
descs []v1.Descriptor descs []registry.Descriptor
) )
if p.git != nil && p.gitRepo != "" { if p.git != nil && p.gitRepo != "" {
@ -360,9 +354,9 @@ func (p Pkg) Build(bos ...BuildOpt) error {
} }
// buildArch builds the package for a single arch // buildArch builds the package for a single arch
func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*v1.Descriptor, error) { func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*registry.Descriptor, error) {
var ( var (
desc *v1.Descriptor desc *registry.Descriptor
tagArch string tagArch string
tag = p.Tag() tag = p.Tag()
) )

View File

@ -13,7 +13,7 @@ import (
"testing" "testing"
"github.com/containerd/containerd/reference" "github.com/containerd/containerd/reference"
"github.com/google/go-containerregistry/pkg/v1" registry "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/types"
lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
imagespec "github.com/opencontainers/image-spec/specs-go/v1" imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@ -97,7 +97,7 @@ type cacheMocker struct {
enableImagePull bool enableImagePull bool
enableImageLoad bool enableImageLoad bool
enableIndexWrite bool enableIndexWrite bool
images map[string][]v1.Descriptor images map[string][]registry.Descriptor
hashes map[string][]byte hashes map[string][]byte
} }
@ -126,15 +126,15 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
if err != nil { if err != nil {
return nil, fmt.Errorf("error reading data: %v", err) return nil, fmt.Errorf("error reading data: %v", err)
} }
hash, size, err := v1.SHA256(bytes.NewReader(b)) hash, size, err := registry.SHA256(bytes.NewReader(b))
if err != nil { if err != nil {
return nil, fmt.Errorf("error calculating hash of layer: %v", err) return nil, fmt.Errorf("error calculating hash of layer: %v", err)
} }
c.assignHash(hash.String(), b) c.assignHash(hash.String(), b)
im := v1.Manifest{ im := registry.Manifest{
MediaType: types.OCIManifestSchema1, MediaType: types.OCIManifestSchema1,
Layers: []v1.Descriptor{ Layers: []registry.Descriptor{
{MediaType: types.OCILayer, Size: size, Digest: hash}, {MediaType: types.OCILayer, Size: size, Digest: hash},
}, },
SchemaVersion: 2, SchemaVersion: 2,
@ -145,12 +145,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to marshal new image to json: %v", err) return nil, fmt.Errorf("unable to marshal new image to json: %v", err)
} }
hash, size, err = v1.SHA256(bytes.NewReader(b)) hash, size, err = registry.SHA256(bytes.NewReader(b))
if err != nil { if err != nil {
return nil, fmt.Errorf("error calculating hash of index json: %v", err) return nil, fmt.Errorf("error calculating hash of index json: %v", err)
} }
c.assignHash(hash.String(), b) c.assignHash(hash.String(), b)
desc := v1.Descriptor{ desc := registry.Descriptor{
MediaType: types.OCIManifestSchema1, MediaType: types.OCIManifestSchema1,
Size: size, Size: size,
Digest: hash, Digest: hash,
@ -163,12 +163,12 @@ func (c *cacheMocker) imageWriteStream(ref *reference.Spec, architecture string,
return c.NewSource(ref, "", &desc), nil return c.NewSource(ref, "", &desc), nil
} }
func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) { func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) {
if !c.enableIndexWrite { if !c.enableIndexWrite {
return nil, errors.New("disabled") return nil, errors.New("disabled")
} }
image := ref.String() image := ref.String()
im := v1.IndexManifest{ im := registry.IndexManifest{
MediaType: types.OCIImageIndex, MediaType: types.OCIImageIndex,
Manifests: descriptors, Manifests: descriptors,
SchemaVersion: 2, SchemaVersion: 2,
@ -179,12 +179,12 @@ func (c *cacheMocker) IndexWrite(ref *reference.Spec, descriptors ...v1.Descript
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to marshal new index to json: %v", err) return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
} }
hash, size, err := v1.SHA256(bytes.NewReader(b)) hash, size, err := registry.SHA256(bytes.NewReader(b))
if err != nil { if err != nil {
return nil, fmt.Errorf("error calculating hash of index json: %v", err) return nil, fmt.Errorf("error calculating hash of index json: %v", err)
} }
c.assignHash(hash.String(), b) c.assignHash(hash.String(), b)
desc := v1.Descriptor{ desc := registry.Descriptor{
MediaType: types.OCIImageIndex, MediaType: types.OCIImageIndex,
Size: size, Size: size,
Digest: hash, Digest: hash,
@ -206,13 +206,13 @@ func (c *cacheMocker) Push(name string) error {
return nil return nil
} }
func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Descriptor) (lktspec.ImageSource, error) { func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...registry.Descriptor) (lktspec.ImageSource, error) {
if !c.enabledDescriptorWrite { if !c.enabledDescriptorWrite {
return nil, errors.New("descriptor disabled") return nil, errors.New("descriptor disabled")
} }
var ( var (
image = ref.String() image = ref.String()
im = v1.IndexManifest{ im = registry.IndexManifest{
MediaType: types.OCIImageIndex, MediaType: types.OCIImageIndex,
Manifests: descriptors, Manifests: descriptors,
SchemaVersion: 2, SchemaVersion: 2,
@ -223,12 +223,12 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to marshal new index to json: %v", err) return nil, fmt.Errorf("unable to marshal new index to json: %v", err)
} }
hash, size, err := v1.SHA256(bytes.NewReader(b)) hash, size, err := registry.SHA256(bytes.NewReader(b))
if err != nil { if err != nil {
return nil, fmt.Errorf("error calculating hash of index json: %v", err) return nil, fmt.Errorf("error calculating hash of index json: %v", err)
} }
c.assignHash(hash.String(), b) c.assignHash(hash.String(), b)
root := v1.Descriptor{ root := registry.Descriptor{
MediaType: types.OCIImageIndex, MediaType: types.OCIImageIndex,
Size: size, Size: size,
Digest: hash, Digest: hash,
@ -240,13 +240,13 @@ func (c *cacheMocker) DescriptorWrite(ref *reference.Spec, descriptors ...v1.Des
return c.NewSource(ref, "", &root), nil return c.NewSource(ref, "", &root), nil
} }
func (c *cacheMocker) FindDescriptor(name string) (*v1.Descriptor, error) { func (c *cacheMocker) FindDescriptor(name string) (*registry.Descriptor, error) {
if desc, ok := c.images[name]; ok && len(desc) > 0 { if desc, ok := c.images[name]; ok && len(desc) > 0 {
return &desc[0], nil return &desc[0], nil
} }
return nil, fmt.Errorf("not found %s", name) return nil, fmt.Errorf("not found %s", name)
} }
func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *v1.Descriptor) lktspec.ImageSource { func (c *cacheMocker) NewSource(ref *reference.Spec, architecture string, descriptor *registry.Descriptor) lktspec.ImageSource {
return cacheMockerSource{c, ref, architecture, descriptor} return cacheMockerSource{c, ref, architecture, descriptor}
} }
func (c *cacheMocker) assignHash(hash string, b []byte) { func (c *cacheMocker) assignHash(hash string, b []byte) {
@ -255,9 +255,9 @@ func (c *cacheMocker) assignHash(hash string, b []byte) {
} }
c.hashes[hash] = b c.hashes[hash] = b
} }
func (c *cacheMocker) appendImage(image string, root v1.Descriptor) { func (c *cacheMocker) appendImage(image string, root registry.Descriptor) {
if c.images == nil { if c.images == nil {
c.images = map[string][]v1.Descriptor{} c.images = map[string][]registry.Descriptor{}
} }
c.images[image] = append(c.images[image], root) c.images[image] = append(c.images[image], root)
} }
@ -266,7 +266,7 @@ type cacheMockerSource struct {
c *cacheMocker c *cacheMocker
ref *reference.Spec ref *reference.Spec
architecture string architecture string
descriptor *v1.Descriptor descriptor *registry.Descriptor
} }
func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) { func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) {
@ -275,7 +275,7 @@ func (c cacheMockerSource) Config() (imagespec.ImageConfig, error) {
func (c cacheMockerSource) TarReader() (io.ReadCloser, error) { func (c cacheMockerSource) TarReader() (io.ReadCloser, error) {
return nil, errors.New("unsupported") return nil, errors.New("unsupported")
} }
func (c cacheMockerSource) Descriptor() *v1.Descriptor { func (c cacheMockerSource) Descriptor() *registry.Descriptor {
return c.descriptor return c.descriptor
} }
@ -299,7 +299,6 @@ func TestBuild(t *testing.T) {
err string err string
}{ }{
{"invalid tag", Pkg{image: "docker.io/foo/bar:abc:def:ghi"}, nil, nil, &dockerMocker{}, &cacheMocker{}, "could not resolve references"}, {"invalid tag", Pkg{image: "docker.io/foo/bar:abc:def:ghi"}, nil, nil, &dockerMocker{}, &cacheMocker{}, "could not resolve references"},
{"mismatched platforms", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"arm64"}}, nil, []string{"amd64"}, nil, nil, fmt.Sprintf("arch %s not supported", "amd64")},
{"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"}, {"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"},
{"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must provide linuxkit build cache"}, {"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must provide linuxkit build cache"},
{"unsupported buildkit", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "buildkit not supported, check docker version"}, {"unsupported buildkit", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "buildkit not supported, check docker version"},

View File

@ -277,6 +277,7 @@ func (p Pkg) Tag() string {
return p.org + "/" + p.image + ":" + t return p.org + "/" + p.image + ":" + t
} }
// FullTag returns a reference expanded tag
func (p Pkg) FullTag() string { func (p Pkg) FullTag() string {
return util.ReferenceExpand(p.Tag()) return util.ReferenceExpand(p.Tag())
} }