Merge pull request #3658 from deitch/multiple-build-push

Build or push multiple packages at once

src/cmd/linuxkit/pkg_build.go

@@ -16,10 +16,18 @@ const (
 )
 
 func pkgBuild(args []string) {
+	pkgBuildPush(args, false)
+}
+
+func pkgBuildPush(args []string, withPush bool) {
 	flags := flag.NewFlagSet("pkg build", flag.ExitOnError)
 	flags.Usage = func() {
 		invoked := filepath.Base(os.Args[0])
-		fmt.Fprintf(os.Stderr, "USAGE: %s pkg build [options] path\n\n", invoked)
+		name := "build"
+		if withPush {
+			name = "push"
+		}
+		fmt.Fprintf(os.Stderr, "USAGE: %s pkg %s [options] path\n\n", name, invoked)
 		fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n")
 		fmt.Fprintf(os.Stderr, "\n")
 		flags.PrintDefaults()
@@ -32,19 +40,46 @@ func pkgBuild(args []string) {
 	builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
 	buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
 
-	p, err := pkglib.NewFromCLI(flags, args...)
+	var (
+		release                  *string
+		nobuild, manifest, image *bool
+		imageRef                 = false
+	)
+	image = &imageRef
+	if withPush {
+		release = flags.String("release", "", "Release the given version")
+		nobuild = flags.Bool("nobuild", false, "Skip the build")
+		manifest = flags.Bool("manifest", true, "Create and push multi-arch manifest")
+		image = flags.Bool("image", true, "Build and push image for the current platform")
+	}
+
+	pkgs, err := pkglib.NewFromCLI(flags, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
 
-	fmt.Printf("Building %q\n", p.Tag())
+	var opts []pkglib.BuildOpt
-
+	if *image {
-	opts := []pkglib.BuildOpt{pkglib.WithBuildImage()}
+		opts = append(opts, pkglib.WithBuildImage())
+	}
 	if *force {
 		opts = append(opts, pkglib.WithBuildForce())
 	}
 	opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir))
+
+	if withPush {
+		opts = append(opts, pkglib.WithBuildPush())
+		if *nobuild {
+			opts = append(opts, pkglib.WithBuildSkip())
+		}
+		if *release != "" {
+			opts = append(opts, pkglib.WithRelease(*release))
+		}
+		if *manifest {
+			opts = append(opts, pkglib.WithBuildManifest())
+		}
+	}
 	if *docker {
 		opts = append(opts, pkglib.WithBuildTargetDockerCache())
 	}
@@ -61,21 +96,16 @@ func pkgBuild(args []string) {
 			skipPlatformsMap[strings.Trim(parts[1], " ")] = true
 		}
 	}
-	// if platforms requested is blank, use all from the config
+	// if requested specific platforms, build those. If not, then we will
+	// retrieve the defaults in the loop over each package.
 	var plats []imagespec.Platform
-	if *platforms == "" {
-		for _, a := range p.Arches() {
-			if _, ok := skipPlatformsMap[a]; ok {
-				continue
-			}
-			plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
-		}
-	} else {
 	// don't allow the use of --skip-platforms with --platforms
-		if *skipPlatforms != "" {
+	if *platforms != "" && *skipPlatforms != "" {
 		fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together")
 		os.Exit(1)
 	}
+	// process the platforms if provided
+	if *platforms != "" {
 		for _, p := range strings.Split(*platforms, ",") {
 			parts := strings.SplitN(p, "/", 2)
 			if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
@@ -85,7 +115,6 @@ func pkgBuild(args []string) {
 			plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
 		}
 	}
-	opts = append(opts, pkglib.WithBuildPlatforms(plats...))
 
 	// build the builders map
 	buildersMap := map[string]string{}
@@ -102,10 +131,46 @@ func pkgBuild(args []string) {
 		os.Exit(1)
 	}
 	opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
-	if err := p.Build(opts...); err != nil {
+
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+	for _, p := range pkgs {
+		// things we need our own copies of
+		var (
+			pkgOpts  = make([]pkglib.BuildOpt, len(opts))
+			pkgPlats = make([]imagespec.Platform, len(plats))
+		)
+		copy(pkgOpts, opts)
+		copy(pkgPlats, plats)
+		// unless overridden, platforms are specific to a package, so this needs to be inside the for loop
+		if len(pkgPlats) == 0 {
+			for _, a := range p.Arches() {
+				if _, ok := skipPlatformsMap[a]; ok {
+					continue
+				}
+				pkgPlats = append(pkgPlats, imagespec.Platform{OS: "linux", Architecture: a})
+			}
+		}
+		pkgOpts = append(pkgOpts, pkglib.WithBuildPlatforms(pkgPlats...))
+
+		var msg, action string
+		switch {
+		case !withPush:
+			msg = fmt.Sprintf("Building %q", p.Tag())
+			action = "building"
+		case *nobuild:
+			msg = fmt.Sprintf("Pushing %q without building", p.Tag())
+			action = "building and pushing"
+		default:
+			msg = fmt.Sprintf("Building and pushing %q", p.Tag())
+			action = "building and pushing"
+		}
+
+		fmt.Println(msg)
+
+		if err := p.Build(pkgOpts...); err != nil {
+			fmt.Fprintf(os.Stderr, "Error %s %q: %v\n", action, p.Tag(), err)
 			os.Exit(1)
 		}
+	}
 }
 
 func buildPlatformBuildersMap(inputs string, existing map[string]string) (map[string]string, error) {

src/cmd/linuxkit/pkg_push.go

@@ -1,104 +1,5 @@
 package main
 
-import (
-	"flag"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-)
-
 func pkgPush(args []string) {
-	flags := flag.NewFlagSet("pkg push", flag.ExitOnError)
+	pkgBuildPush(args, true)
-	flags.Usage = func() {
-		invoked := filepath.Base(os.Args[0])
-		fmt.Fprintf(os.Stderr, "USAGE: %s pkg push [options] path\n\n", invoked)
-		fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n")
-		fmt.Fprintf(os.Stderr, "\n")
-		flags.PrintDefaults()
-	}
-
-	force := flags.Bool("force", false, "Force rebuild")
-	release := flags.String("release", "", "Release the given version")
-	nobuild := flags.Bool("nobuild", false, "Skip the build")
-	docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
-	platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
-	builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
-	manifest := flags.Bool("manifest", true, "Create and push multi-arch manifest")
-	image := flags.Bool("image", true, "Build and push image for the current platform")
-	buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
-
-	p, err := pkglib.NewFromCLI(flags, args...)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-
-	var opts []pkglib.BuildOpt
-	opts = append(opts, pkglib.WithBuildPush())
-	if *force {
-		opts = append(opts, pkglib.WithBuildForce())
-	}
-	if *nobuild {
-		opts = append(opts, pkglib.WithBuildSkip())
-	}
-	if *release != "" {
-		opts = append(opts, pkglib.WithRelease(*release))
-	}
-	if *manifest {
-		opts = append(opts, pkglib.WithBuildManifest())
-	}
-	if *image {
-		opts = append(opts, pkglib.WithBuildImage())
-	}
-	opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir))
-	if *docker {
-		opts = append(opts, pkglib.WithBuildTargetDockerCache())
-	}
-	// if platforms requested is blank, use all from the config
-	var plats []imagespec.Platform
-	if *platforms == "" {
-		for _, a := range p.Arches() {
-			plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
-		}
-	} else {
-		for _, p := range strings.Split(*platforms, ",") {
-			parts := strings.SplitN(p, "/", 2)
-			if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
-				fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", p)
-				os.Exit(1)
-			}
-			plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
-		}
-	}
-	opts = append(opts, pkglib.WithBuildPlatforms(plats...))
-	// build the builders map
-	buildersMap := map[string]string{}
-	// look for builders env var
-	buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s in environment variable %s\n", err.Error(), buildersEnvVar)
-		os.Exit(1)
-	}
-	// any CLI options override env var
-	buildersMap, err = buildPlatformBuildersMap(*builders, buildersMap)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s in --builders flag\n", err.Error())
-		os.Exit(1)
-	}
-	opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
-
-	if *nobuild {
-		fmt.Printf("Pushing %q without building\n", p.Tag())
-	} else {
-		fmt.Printf("Building and pushing %q\n", p.Tag())
-	}
-
-	if err := p.Build(opts...); err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
 }

src/cmd/linuxkit/pkg_showtag.go

@@ -19,11 +19,12 @@ func pkgShowTag(args []string) {
 		flags.PrintDefaults()
 	}
 
-	p, err := pkglib.NewFromCLI(flags, args...)
+	pkgs, err := pkglib.NewFromCLI(flags, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
-
+	for _, p := range pkgs {
 		fmt.Println(p.Tag())
+	}
 }

src/cmd/linuxkit/pkglib/pkglib.go

@@ -64,10 +64,10 @@ type Pkg struct {
 	git        *git
 }
 
-// NewFromCLI creates a Pkg from a set of CLI arguments. Calls fs.Parse()
+// NewFromCLI creates a range of Pkg from a set of CLI arguments. Calls fs.Parse()
-func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
+func NewFromCLI(fs *flag.FlagSet, args ...string) ([]Pkg, error) {
 	// Defaults
-	pi := pkgInfo{
+	piBase := pkgInfo{
 		Org:          "linuxkit",
 		Arches:       []string{"amd64", "arm64", "s390x"},
 		GitRepo:      "https://github.com/linuxkit/linuxkit",
@@ -80,69 +80,84 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 	// Ideally want to look at every directory from root to `pkg`
 	// for this file but might be tricky to arrange ordering-wise.
 
-	// These override fields in pi below, bools are in both forms to allow user overrides in either direction
+	// These override fields in pi below, bools are in both forms to allow user overrides in either direction.
-	argDisableCache := fs.Bool("disable-cache", pi.DisableCache, "Disable build cache")
+	// These will apply to all packages built.
-	argEnableCache := fs.Bool("enable-cache", !pi.DisableCache, "Enable build cache")
+	argDisableCache := fs.Bool("disable-cache", piBase.DisableCache, "Disable build cache")
-	argNoNetwork := fs.Bool("nonetwork", !pi.Network, "Disallow network use during build")
+	argEnableCache := fs.Bool("enable-cache", !piBase.DisableCache, "Enable build cache")
-	argNetwork := fs.Bool("network", pi.Network, "Allow network use during build")
+	argNoNetwork := fs.Bool("nonetwork", !piBase.Network, "Disallow network use during build")
+	argNetwork := fs.Bool("network", piBase.Network, "Allow network use during build")
 
-	argOrg := fs.String("org", pi.Org, "Override the hub org")
+	argOrg := fs.String("org", piBase.Org, "Override the hub org")
 
 	// Other arguments
 	var buildYML, hash, hashCommit, hashPath string
 	var dirty, devMode bool
+
 	fs.StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file")
 	fs.StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)")
 	fs.StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash")
 	fs.StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)")
-	fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg to be considered dirty")
+	fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty")
 	fs.BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively")
 
-	fs.Parse(args)
+	_ = fs.Parse(args)
 
 	if fs.NArg() < 1 {
-		return Pkg{}, fmt.Errorf("A pkg directory is required")
+		return nil, fmt.Errorf("At least one pkg directory is required")
-	}
-	if fs.NArg() > 1 {
-		return Pkg{}, fmt.Errorf("Unknown extra arguments given: %s", fs.Args()[1:])
 	}
 
-	pkg := fs.Arg(0)
+	var pkgs []Pkg
+	for _, pkg := range fs.Args() {
+		var (
+			pkgHashPath string
+			pkgHash     = hash
+		)
 		pkgPath, err := filepath.Abs(pkg)
 		if err != nil {
-		return Pkg{}, err
+			return nil, err
 		}
 
 		if hashPath == "" {
-		hashPath = pkgPath
+			pkgHashPath = pkgPath
 		} else {
-		hashPath, err = filepath.Abs(hashPath)
+			pkgHashPath, err = filepath.Abs(hashPath)
 			if err != nil {
-			return Pkg{}, err
+				return nil, err
 			}
 
-		if !strings.HasPrefix(pkgPath, hashPath) {
+			if !strings.HasPrefix(pkgPath, pkgHashPath) {
-			return Pkg{}, fmt.Errorf("Hash path is not a prefix of the package path")
+				return nil, fmt.Errorf("Hash path is not a prefix of the package path")
 			}
 
 			// TODO(ijc) pkgPath and hashPath really ought to be in the same git tree too...
 		}
 
+		// make our own copy of piBase. We could use some deepcopy library, but it is just as easy to marshal/unmarshal
+		pib, err := yaml.Marshal(&piBase)
+		if err != nil {
+			return nil, err
+		}
+		var pi pkgInfo
+		if err := yaml.Unmarshal(pib, &pi); err != nil {
+			return nil, err
+		}
+
 		b, err := ioutil.ReadFile(filepath.Join(pkgPath, buildYML))
 		if err != nil {
-		return Pkg{}, err
+			return nil, err
 		}
+
 		if err := yaml.Unmarshal(b, &pi); err != nil {
-		return Pkg{}, err
+			return nil, err
 		}
 
 		if pi.Image == "" {
-		return Pkg{}, fmt.Errorf("Image field is required")
+			return nil, fmt.Errorf("Image field is required")
 		}
 
 		dockerDepends, err := newDockerDepends(pkgPath, &pi)
 		if err != nil {
-		return Pkg{}, err
+			return nil, err
 		}
 
 		if devMode {
@@ -150,14 +165,15 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 			// by argOrg when we iterate over the provided options
 			// in the fs.Visit block below.
 			pi.Org = os.Getenv("USER")
-		if hash == "" {
+			if pkgHash == "" {
-			hash = "dev"
+				pkgHash = "dev"
 			}
 		}
 
 		// Go's flag package provides no way to see if a flag was set
 		// apart from Visit which iterates over only those which were
-	// set.
+		// set. This must be run here, rather than earlier, because we need to
+		// have read it from the build.yml file first, then override based on CLI.
 		fs.Visit(func(f *flag.Flag) {
 			switch f.Name {
 			case "disable-cache":
@@ -179,7 +195,7 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 		for _, source := range pi.ExtraSources {
 			tmp := strings.Split(source, ":")
 			if len(tmp) != 2 {
-			return Pkg{}, fmt.Errorf("Bad source format in %s", source)
+				return nil, fmt.Errorf("Bad source format in %s", source)
 			}
 			srcPath := filepath.Clean(tmp[0]) // Should work with windows paths
 			dstPath := path.Clean(tmp[1])     // 'path' here because this should be a Unix path
@@ -190,14 +206,14 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 
 			g, err := newGit(srcPath)
 			if err != nil {
-			return Pkg{}, err
+				return nil, err
 			}
 			if g == nil {
-			return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath)
+				return nil, fmt.Errorf("Source %s not in a git repository", srcPath)
 			}
 			h, err := g.treeHash(srcPath, hashCommit)
 			if err != nil {
-			return Pkg{}, err
+				return nil, err
 			}
 
 			srcHashes += h
@@ -206,37 +222,37 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 
 		git, err := newGit(pkgPath)
 		if err != nil {
-		return Pkg{}, err
+			return nil, err
 		}
 
 		if git != nil {
-		gitDirty, err := git.isDirty(hashPath, hashCommit)
+			gitDirty, err := git.isDirty(pkgHashPath, hashCommit)
 			if err != nil {
-			return Pkg{}, err
+				return nil, err
 			}
 
 			dirty = dirty || gitDirty
 
-		if hash == "" {
+			if pkgHash == "" {
-			if hash, err = git.treeHash(hashPath, hashCommit); err != nil {
+				if pkgHash, err = git.treeHash(pkgHashPath, hashCommit); err != nil {
-				return Pkg{}, err
+					return nil, err
 				}
 
 				if srcHashes != "" {
-				hash += srcHashes
+					pkgHash += srcHashes
-				hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash)))
+					pkgHash = fmt.Sprintf("%x", sha1.Sum([]byte(pkgHash)))
 				}
 
 				if dirty {
-				hash += "-dirty"
+					pkgHash += "-dirty"
 				}
 			}
 		}
 
-	return Pkg{
+		pkgs = append(pkgs, Pkg{
 			image:         pi.Image,
 			org:           pi.Org,
-		hash:          hash,
+			hash:          pkgHash,
 			commitHash:    hashCommit,
 			arches:        pi.Arches,
 			sources:       sources,
@@ -248,7 +264,9 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 			dirty:         dirty,
 			path:          pkgPath,
 			git:           git,
-	}, nil
+		})
+	}
+	return pkgs, nil
 }
 
 // Hash returns the hash of the package

src/cmd/linuxkit/pkglib/pkglib_test.go

@@ -39,8 +39,9 @@ func testBool(t *testing.T, key string, inv bool, forceOn, forceOff string, get
 				args = append(args, override)
 			}
 			args = append(args, pkgDir)
-			pkg, err := NewFromCLI(flags, args...)
+			pkgs, err := NewFromCLI(flags, args...)
 			require.NoError(t, err)
+			pkg := pkgs[0]
 			t.Logf("override %q produced %t", override, get(pkg))
 			f(t, pkg)
 		}