Build or push multiple packages at once

Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
Avi Deitcher 2021-05-04 21:55:34 -04:00
parent be2813f51f
commit 31ed260e4a
5 changed files with 260 additions and 274 deletions

View File

@ -16,10 +16,18 @@ const (
)
func pkgBuild(args []string) {
pkgBuildPush(args, false)
}
func pkgBuildPush(args []string, withPush bool) {
flags := flag.NewFlagSet("pkg build", flag.ExitOnError)
flags.Usage = func() {
invoked := filepath.Base(os.Args[0])
fmt.Fprintf(os.Stderr, "USAGE: %s pkg build [options] path\n\n", invoked)
name := "build"
if withPush {
name = "push"
}
fmt.Fprintf(os.Stderr, "USAGE: %s pkg %s [options] path\n\n", name, invoked)
fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n")
fmt.Fprintf(os.Stderr, "\n")
flags.PrintDefaults()
@ -32,19 +40,46 @@ func pkgBuild(args []string) {
builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
p, err := pkglib.NewFromCLI(flags, args...)
var (
release *string
nobuild, manifest, image *bool
imageRef = false
)
image = &imageRef
if withPush {
release = flags.String("release", "", "Release the given version")
nobuild = flags.Bool("nobuild", false, "Skip the build")
manifest = flags.Bool("manifest", true, "Create and push multi-arch manifest")
image = flags.Bool("image", true, "Build and push image for the current platform")
}
pkgs, err := pkglib.NewFromCLI(flags, args...)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
fmt.Printf("Building %q\n", p.Tag())
opts := []pkglib.BuildOpt{pkglib.WithBuildImage()}
var opts []pkglib.BuildOpt
if *image {
opts = append(opts, pkglib.WithBuildImage())
}
if *force {
opts = append(opts, pkglib.WithBuildForce())
}
opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir))
if withPush {
opts = append(opts, pkglib.WithBuildPush())
if *nobuild {
opts = append(opts, pkglib.WithBuildSkip())
}
if *release != "" {
opts = append(opts, pkglib.WithRelease(*release))
}
if *manifest {
opts = append(opts, pkglib.WithBuildManifest())
}
}
if *docker {
opts = append(opts, pkglib.WithBuildTargetDockerCache())
}
@ -61,21 +96,16 @@ func pkgBuild(args []string) {
skipPlatformsMap[strings.Trim(parts[1], " ")] = true
}
}
// if platforms requested is blank, use all from the config
// if requested specific platforms, build those. If not, then we will
// retrieve the defaults in the loop over each package.
var plats []imagespec.Platform
if *platforms == "" {
for _, a := range p.Arches() {
if _, ok := skipPlatformsMap[a]; ok {
continue
}
plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
}
} else {
// don't allow the use of --skip-platforms with --platforms
if *skipPlatforms != "" {
fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together")
os.Exit(1)
}
// don't allow the use of --skip-platforms with --platforms
if *platforms != "" && *skipPlatforms != "" {
fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together")
os.Exit(1)
}
// process the platforms if provided
if *platforms != "" {
for _, p := range strings.Split(*platforms, ",") {
parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
@ -85,7 +115,6 @@ func pkgBuild(args []string) {
plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
}
}
opts = append(opts, pkglib.WithBuildPlatforms(plats...))
// build the builders map
buildersMap := map[string]string{}
@ -102,9 +131,45 @@ func pkgBuild(args []string) {
os.Exit(1)
}
opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
if err := p.Build(opts...); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
for _, p := range pkgs {
// things we need our own copies of
var (
pkgOpts = make([]pkglib.BuildOpt, len(opts))
pkgPlats = make([]imagespec.Platform, len(plats))
)
copy(pkgOpts, opts)
copy(pkgPlats, plats)
// unless overridden, platforms are specific to a package, so this needs to be inside the for loop
if len(pkgPlats) == 0 {
for _, a := range p.Arches() {
if _, ok := skipPlatformsMap[a]; ok {
continue
}
pkgPlats = append(pkgPlats, imagespec.Platform{OS: "linux", Architecture: a})
}
}
pkgOpts = append(pkgOpts, pkglib.WithBuildPlatforms(pkgPlats...))
var msg, action string
switch {
case !withPush:
msg = fmt.Sprintf("Building %q", p.Tag())
action = "building"
case *nobuild:
msg = fmt.Sprintf("Pushing %q without building", p.Tag())
action = "building and pushing"
default:
msg = fmt.Sprintf("Building and pushing %q", p.Tag())
action = "building and pushing"
}
fmt.Println(msg)
if err := p.Build(pkgOpts...); err != nil {
fmt.Fprintf(os.Stderr, "Error %s %q: %v\n", action, p.Tag(), err)
os.Exit(1)
}
}
}

View File

@ -1,104 +1,5 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
func pkgPush(args []string) {
flags := flag.NewFlagSet("pkg push", flag.ExitOnError)
flags.Usage = func() {
invoked := filepath.Base(os.Args[0])
fmt.Fprintf(os.Stderr, "USAGE: %s pkg push [options] path\n\n", invoked)
fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n")
fmt.Fprintf(os.Stderr, "\n")
flags.PrintDefaults()
}
force := flags.Bool("force", false, "Force rebuild")
release := flags.String("release", "", "Release the given version")
nobuild := flags.Bool("nobuild", false, "Skip the build")
docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
manifest := flags.Bool("manifest", true, "Create and push multi-arch manifest")
image := flags.Bool("image", true, "Build and push image for the current platform")
buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker")
p, err := pkglib.NewFromCLI(flags, args...)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
var opts []pkglib.BuildOpt
opts = append(opts, pkglib.WithBuildPush())
if *force {
opts = append(opts, pkglib.WithBuildForce())
}
if *nobuild {
opts = append(opts, pkglib.WithBuildSkip())
}
if *release != "" {
opts = append(opts, pkglib.WithRelease(*release))
}
if *manifest {
opts = append(opts, pkglib.WithBuildManifest())
}
if *image {
opts = append(opts, pkglib.WithBuildImage())
}
opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir))
if *docker {
opts = append(opts, pkglib.WithBuildTargetDockerCache())
}
// if platforms requested is blank, use all from the config
var plats []imagespec.Platform
if *platforms == "" {
for _, a := range p.Arches() {
plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a})
}
} else {
for _, p := range strings.Split(*platforms, ",") {
parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", p)
os.Exit(1)
}
plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]})
}
}
opts = append(opts, pkglib.WithBuildPlatforms(plats...))
// build the builders map
buildersMap := map[string]string{}
// look for builders env var
buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap)
if err != nil {
fmt.Fprintf(os.Stderr, "%s in environment variable %s\n", err.Error(), buildersEnvVar)
os.Exit(1)
}
// any CLI options override env var
buildersMap, err = buildPlatformBuildersMap(*builders, buildersMap)
if err != nil {
fmt.Fprintf(os.Stderr, "%s in --builders flag\n", err.Error())
os.Exit(1)
}
opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
if *nobuild {
fmt.Printf("Pushing %q without building\n", p.Tag())
} else {
fmt.Printf("Building and pushing %q\n", p.Tag())
}
if err := p.Build(opts...); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
pkgBuildPush(args, true)
}

View File

@ -19,11 +19,12 @@ func pkgShowTag(args []string) {
flags.PrintDefaults()
}
p, err := pkglib.NewFromCLI(flags, args...)
pkgs, err := pkglib.NewFromCLI(flags, args...)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
fmt.Println(p.Tag())
for _, p := range pkgs {
fmt.Println(p.Tag())
}
}

View File

@ -64,10 +64,10 @@ type Pkg struct {
git *git
}
// NewFromCLI creates a Pkg from a set of CLI arguments. Calls fs.Parse()
func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
// NewFromCLI creates a range of Pkg from a set of CLI arguments. Calls fs.Parse()
func NewFromCLI(fs *flag.FlagSet, args ...string) ([]Pkg, error) {
// Defaults
pi := pkgInfo{
piBase := pkgInfo{
Org: "linuxkit",
Arches: []string{"amd64", "arm64", "s390x"},
GitRepo: "https://github.com/linuxkit/linuxkit",
@ -80,175 +80,193 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
// Ideally want to look at every directory from root to `pkg`
// for this file but might be tricky to arrange ordering-wise.
// These override fields in pi below, bools are in both forms to allow user overrides in either direction
argDisableCache := fs.Bool("disable-cache", pi.DisableCache, "Disable build cache")
argEnableCache := fs.Bool("enable-cache", !pi.DisableCache, "Enable build cache")
argNoNetwork := fs.Bool("nonetwork", !pi.Network, "Disallow network use during build")
argNetwork := fs.Bool("network", pi.Network, "Allow network use during build")
// These override fields in pi below, bools are in both forms to allow user overrides in either direction.
// These will apply to all packages built.
argDisableCache := fs.Bool("disable-cache", piBase.DisableCache, "Disable build cache")
argEnableCache := fs.Bool("enable-cache", !piBase.DisableCache, "Enable build cache")
argNoNetwork := fs.Bool("nonetwork", !piBase.Network, "Disallow network use during build")
argNetwork := fs.Bool("network", piBase.Network, "Allow network use during build")
argOrg := fs.String("org", pi.Org, "Override the hub org")
argOrg := fs.String("org", piBase.Org, "Override the hub org")
// Other arguments
var buildYML, hash, hashCommit, hashPath string
var dirty, devMode bool
fs.StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file")
fs.StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)")
fs.StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash")
fs.StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)")
fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg to be considered dirty")
fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty")
fs.BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively")
fs.Parse(args)
_ = fs.Parse(args)
if fs.NArg() < 1 {
return Pkg{}, fmt.Errorf("A pkg directory is required")
}
if fs.NArg() > 1 {
return Pkg{}, fmt.Errorf("Unknown extra arguments given: %s", fs.Args()[1:])
return nil, fmt.Errorf("At least one pkg directory is required")
}
pkg := fs.Arg(0)
pkgPath, err := filepath.Abs(pkg)
if err != nil {
return Pkg{}, err
}
if hashPath == "" {
hashPath = pkgPath
} else {
hashPath, err = filepath.Abs(hashPath)
var pkgs []Pkg
for _, pkg := range fs.Args() {
var (
pkgHashPath string
pkgHash = hash
)
pkgPath, err := filepath.Abs(pkg)
if err != nil {
return Pkg{}, err
return nil, err
}
if !strings.HasPrefix(pkgPath, hashPath) {
return Pkg{}, fmt.Errorf("Hash path is not a prefix of the package path")
}
// TODO(ijc) pkgPath and hashPath really ought to be in the same git tree too...
}
b, err := ioutil.ReadFile(filepath.Join(pkgPath, buildYML))
if err != nil {
return Pkg{}, err
}
if err := yaml.Unmarshal(b, &pi); err != nil {
return Pkg{}, err
}
if pi.Image == "" {
return Pkg{}, fmt.Errorf("Image field is required")
}
dockerDepends, err := newDockerDepends(pkgPath, &pi)
if err != nil {
return Pkg{}, err
}
if devMode {
// If --org is also used then this will be overwritten
// by argOrg when we iterate over the provided options
// in the fs.Visit block below.
pi.Org = os.Getenv("USER")
if hash == "" {
hash = "dev"
}
}
// Go's flag package provides no way to see if a flag was set
// apart from Visit which iterates over only those which were
// set.
fs.Visit(func(f *flag.Flag) {
switch f.Name {
case "disable-cache":
pi.DisableCache = *argDisableCache
case "enable-cache":
pi.DisableCache = !*argEnableCache
case "network":
pi.Network = *argNetwork
case "nonetwork":
pi.Network = !*argNoNetwork
case "org":
pi.Org = *argOrg
}
})
var srcHashes string
sources := []pkgSource{{src: pkgPath, dst: "/"}}
for _, source := range pi.ExtraSources {
tmp := strings.Split(source, ":")
if len(tmp) != 2 {
return Pkg{}, fmt.Errorf("Bad source format in %s", source)
}
srcPath := filepath.Clean(tmp[0]) // Should work with windows paths
dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path
if !filepath.IsAbs(srcPath) {
srcPath = filepath.Join(pkgPath, srcPath)
}
g, err := newGit(srcPath)
if err != nil {
return Pkg{}, err
}
if g == nil {
return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath)
}
h, err := g.treeHash(srcPath, hashCommit)
if err != nil {
return Pkg{}, err
}
srcHashes += h
sources = append(sources, pkgSource{src: srcPath, dst: dstPath})
}
git, err := newGit(pkgPath)
if err != nil {
return Pkg{}, err
}
if git != nil {
gitDirty, err := git.isDirty(hashPath, hashCommit)
if err != nil {
return Pkg{}, err
}
dirty = dirty || gitDirty
if hash == "" {
if hash, err = git.treeHash(hashPath, hashCommit); err != nil {
return Pkg{}, err
if hashPath == "" {
pkgHashPath = pkgPath
} else {
pkgHashPath, err = filepath.Abs(hashPath)
if err != nil {
return nil, err
}
if srcHashes != "" {
hash += srcHashes
hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash)))
if !strings.HasPrefix(pkgPath, pkgHashPath) {
return nil, fmt.Errorf("Hash path is not a prefix of the package path")
}
if dirty {
hash += "-dirty"
// TODO(ijc) pkgPath and hashPath really ought to be in the same git tree too...
}
// make our own copy of piBase. We could use some deepcopy library, but it is just as easy to marshal/unmarshal
pib, err := yaml.Marshal(&piBase)
if err != nil {
return nil, err
}
var pi pkgInfo
if err := yaml.Unmarshal(pib, &pi); err != nil {
return nil, err
}
b, err := ioutil.ReadFile(filepath.Join(pkgPath, buildYML))
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(b, &pi); err != nil {
return nil, err
}
if pi.Image == "" {
return nil, fmt.Errorf("Image field is required")
}
dockerDepends, err := newDockerDepends(pkgPath, &pi)
if err != nil {
return nil, err
}
if devMode {
// If --org is also used then this will be overwritten
// by argOrg when we iterate over the provided options
// in the fs.Visit block below.
pi.Org = os.Getenv("USER")
if pkgHash == "" {
pkgHash = "dev"
}
}
}
return Pkg{
image: pi.Image,
org: pi.Org,
hash: hash,
commitHash: hashCommit,
arches: pi.Arches,
sources: sources,
gitRepo: pi.GitRepo,
network: pi.Network,
cache: !pi.DisableCache,
config: pi.Config,
dockerDepends: dockerDepends,
dirty: dirty,
path: pkgPath,
git: git,
}, nil
// Go's flag package provides no way to see if a flag was set
// apart from Visit which iterates over only those which were
// set. This must be run here, rather than earlier, because we need to
// have read it from the build.yml file first, then override based on CLI.
fs.Visit(func(f *flag.Flag) {
switch f.Name {
case "disable-cache":
pi.DisableCache = *argDisableCache
case "enable-cache":
pi.DisableCache = !*argEnableCache
case "network":
pi.Network = *argNetwork
case "nonetwork":
pi.Network = !*argNoNetwork
case "org":
pi.Org = *argOrg
}
})
var srcHashes string
sources := []pkgSource{{src: pkgPath, dst: "/"}}
for _, source := range pi.ExtraSources {
tmp := strings.Split(source, ":")
if len(tmp) != 2 {
return nil, fmt.Errorf("Bad source format in %s", source)
}
srcPath := filepath.Clean(tmp[0]) // Should work with windows paths
dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path
if !filepath.IsAbs(srcPath) {
srcPath = filepath.Join(pkgPath, srcPath)
}
g, err := newGit(srcPath)
if err != nil {
return nil, err
}
if g == nil {
return nil, fmt.Errorf("Source %s not in a git repository", srcPath)
}
h, err := g.treeHash(srcPath, hashCommit)
if err != nil {
return nil, err
}
srcHashes += h
sources = append(sources, pkgSource{src: srcPath, dst: dstPath})
}
git, err := newGit(pkgPath)
if err != nil {
return nil, err
}
if git != nil {
gitDirty, err := git.isDirty(pkgHashPath, hashCommit)
if err != nil {
return nil, err
}
dirty = dirty || gitDirty
if pkgHash == "" {
if pkgHash, err = git.treeHash(pkgHashPath, hashCommit); err != nil {
return nil, err
}
if srcHashes != "" {
pkgHash += srcHashes
pkgHash = fmt.Sprintf("%x", sha1.Sum([]byte(pkgHash)))
}
if dirty {
pkgHash += "-dirty"
}
}
}
pkgs = append(pkgs, Pkg{
image: pi.Image,
org: pi.Org,
hash: pkgHash,
commitHash: hashCommit,
arches: pi.Arches,
sources: sources,
gitRepo: pi.GitRepo,
network: pi.Network,
cache: !pi.DisableCache,
config: pi.Config,
dockerDepends: dockerDepends,
dirty: dirty,
path: pkgPath,
git: git,
})
}
return pkgs, nil
}
// Hash returns the hash of the package

View File

@ -39,8 +39,9 @@ func testBool(t *testing.T, key string, inv bool, forceOn, forceOff string, get
args = append(args, override)
}
args = append(args, pkgDir)
pkg, err := NewFromCLI(flags, args...)
pkgs, err := NewFromCLI(flags, args...)
require.NoError(t, err)
pkg := pkgs[0]
t.Logf("override %q produced %t", override, get(pkg))
f(t, pkg)
}