From 31ed260e4a9027da1406f8a2e3f139c5dea69cd5 Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Tue, 4 May 2021 21:55:34 -0400 Subject: [PATCH] Build or push multiple packages at once Signed-off-by: Avi Deitcher --- src/cmd/linuxkit/pkg_build.go | 111 +++++++-- src/cmd/linuxkit/pkg_push.go | 101 +------- src/cmd/linuxkit/pkg_showtag.go | 7 +- src/cmd/linuxkit/pkglib/pkglib.go | 312 +++++++++++++------------ src/cmd/linuxkit/pkglib/pkglib_test.go | 3 +- 5 files changed, 260 insertions(+), 274 deletions(-) diff --git a/src/cmd/linuxkit/pkg_build.go b/src/cmd/linuxkit/pkg_build.go index 86bac742b..657bab092 100644 --- a/src/cmd/linuxkit/pkg_build.go +++ b/src/cmd/linuxkit/pkg_build.go @@ -16,10 +16,18 @@ const ( ) func pkgBuild(args []string) { + pkgBuildPush(args, false) +} + +func pkgBuildPush(args []string, withPush bool) { flags := flag.NewFlagSet("pkg build", flag.ExitOnError) flags.Usage = func() { invoked := filepath.Base(os.Args[0]) - fmt.Fprintf(os.Stderr, "USAGE: %s pkg build [options] path\n\n", invoked) + name := "build" + if withPush { + name = "push" + } + fmt.Fprintf(os.Stderr, "USAGE: %s pkg %s [options] path\n\n", name, invoked) fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n") fmt.Fprintf(os.Stderr, "\n") flags.PrintDefaults() @@ -32,19 +40,46 @@ func pkgBuild(args []string) { builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes") buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker") - p, err := pkglib.NewFromCLI(flags, args...) + var ( + release *string + nobuild, manifest, image *bool + imageRef = false + ) + image = &imageRef + if withPush { + release = flags.String("release", "", "Release the given version") + nobuild = flags.Bool("nobuild", false, "Skip the build") + manifest = flags.Bool("manifest", true, "Create and push multi-arch manifest") + image = flags.Bool("image", true, "Build and push image for the current platform") + } + + pkgs, err := pkglib.NewFromCLI(flags, args...) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } - fmt.Printf("Building %q\n", p.Tag()) - - opts := []pkglib.BuildOpt{pkglib.WithBuildImage()} + var opts []pkglib.BuildOpt + if *image { + opts = append(opts, pkglib.WithBuildImage()) + } if *force { opts = append(opts, pkglib.WithBuildForce()) } opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir)) + + if withPush { + opts = append(opts, pkglib.WithBuildPush()) + if *nobuild { + opts = append(opts, pkglib.WithBuildSkip()) + } + if *release != "" { + opts = append(opts, pkglib.WithRelease(*release)) + } + if *manifest { + opts = append(opts, pkglib.WithBuildManifest()) + } + } if *docker { opts = append(opts, pkglib.WithBuildTargetDockerCache()) } @@ -61,21 +96,16 @@ func pkgBuild(args []string) { skipPlatformsMap[strings.Trim(parts[1], " ")] = true } } - // if platforms requested is blank, use all from the config + // if requested specific platforms, build those. If not, then we will + // retrieve the defaults in the loop over each package. var plats []imagespec.Platform - if *platforms == "" { - for _, a := range p.Arches() { - if _, ok := skipPlatformsMap[a]; ok { - continue - } - plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a}) - } - } else { - // don't allow the use of --skip-platforms with --platforms - if *skipPlatforms != "" { - fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together") - os.Exit(1) - } + // don't allow the use of --skip-platforms with --platforms + if *platforms != "" && *skipPlatforms != "" { + fmt.Fprintln(os.Stderr, "--skip-platforms and --platforms may not be used together") + os.Exit(1) + } + // process the platforms if provided + if *platforms != "" { for _, p := range strings.Split(*platforms, ",") { parts := strings.SplitN(p, "/", 2) if len(parts) != 2 || parts[0] == "" || parts[1] == "" { @@ -85,7 +115,6 @@ func pkgBuild(args []string) { plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]}) } } - opts = append(opts, pkglib.WithBuildPlatforms(plats...)) // build the builders map buildersMap := map[string]string{} @@ -102,9 +131,45 @@ func pkgBuild(args []string) { os.Exit(1) } opts = append(opts, pkglib.WithBuildBuilders(buildersMap)) - if err := p.Build(opts...); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) + + for _, p := range pkgs { + // things we need our own copies of + var ( + pkgOpts = make([]pkglib.BuildOpt, len(opts)) + pkgPlats = make([]imagespec.Platform, len(plats)) + ) + copy(pkgOpts, opts) + copy(pkgPlats, plats) + // unless overridden, platforms are specific to a package, so this needs to be inside the for loop + if len(pkgPlats) == 0 { + for _, a := range p.Arches() { + if _, ok := skipPlatformsMap[a]; ok { + continue + } + pkgPlats = append(pkgPlats, imagespec.Platform{OS: "linux", Architecture: a}) + } + } + pkgOpts = append(pkgOpts, pkglib.WithBuildPlatforms(pkgPlats...)) + + var msg, action string + switch { + case !withPush: + msg = fmt.Sprintf("Building %q", p.Tag()) + action = "building" + case *nobuild: + msg = fmt.Sprintf("Pushing %q without building", p.Tag()) + action = "building and pushing" + default: + msg = fmt.Sprintf("Building and pushing %q", p.Tag()) + action = "building and pushing" + } + + fmt.Println(msg) + + if err := p.Build(pkgOpts...); err != nil { + fmt.Fprintf(os.Stderr, "Error %s %q: %v\n", action, p.Tag(), err) + os.Exit(1) + } } } diff --git a/src/cmd/linuxkit/pkg_push.go b/src/cmd/linuxkit/pkg_push.go index e0094e28b..8ce24a13f 100644 --- a/src/cmd/linuxkit/pkg_push.go +++ b/src/cmd/linuxkit/pkg_push.go @@ -1,104 +1,5 @@ package main -import ( - "flag" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" -) - func pkgPush(args []string) { - flags := flag.NewFlagSet("pkg push", flag.ExitOnError) - flags.Usage = func() { - invoked := filepath.Base(os.Args[0]) - fmt.Fprintf(os.Stderr, "USAGE: %s pkg push [options] path\n\n", invoked) - fmt.Fprintf(os.Stderr, "'path' specifies the path to the package source directory.\n") - fmt.Fprintf(os.Stderr, "\n") - flags.PrintDefaults() - } - - force := flags.Bool("force", false, "Force rebuild") - release := flags.String("release", "", "Release the given version") - nobuild := flags.Bool("nobuild", false, "Skip the build") - docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache") - platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built") - builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes") - manifest := flags.Bool("manifest", true, "Create and push multi-arch manifest") - image := flags.Bool("image", true, "Build and push image for the current platform") - buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker") - - p, err := pkglib.NewFromCLI(flags, args...) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - var opts []pkglib.BuildOpt - opts = append(opts, pkglib.WithBuildPush()) - if *force { - opts = append(opts, pkglib.WithBuildForce()) - } - if *nobuild { - opts = append(opts, pkglib.WithBuildSkip()) - } - if *release != "" { - opts = append(opts, pkglib.WithRelease(*release)) - } - if *manifest { - opts = append(opts, pkglib.WithBuildManifest()) - } - if *image { - opts = append(opts, pkglib.WithBuildImage()) - } - opts = append(opts, pkglib.WithBuildCacheDir(*buildCacheDir)) - if *docker { - opts = append(opts, pkglib.WithBuildTargetDockerCache()) - } - // if platforms requested is blank, use all from the config - var plats []imagespec.Platform - if *platforms == "" { - for _, a := range p.Arches() { - plats = append(plats, imagespec.Platform{OS: "linux", Architecture: a}) - } - } else { - for _, p := range strings.Split(*platforms, ",") { - parts := strings.SplitN(p, "/", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - fmt.Fprintf(os.Stderr, "invalid target platform specification '%s'\n", p) - os.Exit(1) - } - plats = append(plats, imagespec.Platform{OS: parts[0], Architecture: parts[1]}) - } - } - opts = append(opts, pkglib.WithBuildPlatforms(plats...)) - // build the builders map - buildersMap := map[string]string{} - // look for builders env var - buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap) - if err != nil { - fmt.Fprintf(os.Stderr, "%s in environment variable %s\n", err.Error(), buildersEnvVar) - os.Exit(1) - } - // any CLI options override env var - buildersMap, err = buildPlatformBuildersMap(*builders, buildersMap) - if err != nil { - fmt.Fprintf(os.Stderr, "%s in --builders flag\n", err.Error()) - os.Exit(1) - } - opts = append(opts, pkglib.WithBuildBuilders(buildersMap)) - - if *nobuild { - fmt.Printf("Pushing %q without building\n", p.Tag()) - } else { - fmt.Printf("Building and pushing %q\n", p.Tag()) - } - - if err := p.Build(opts...); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } + pkgBuildPush(args, true) } diff --git a/src/cmd/linuxkit/pkg_showtag.go b/src/cmd/linuxkit/pkg_showtag.go index 58cdb814f..6b08da631 100644 --- a/src/cmd/linuxkit/pkg_showtag.go +++ b/src/cmd/linuxkit/pkg_showtag.go @@ -19,11 +19,12 @@ func pkgShowTag(args []string) { flags.PrintDefaults() } - p, err := pkglib.NewFromCLI(flags, args...) + pkgs, err := pkglib.NewFromCLI(flags, args...) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } - - fmt.Println(p.Tag()) + for _, p := range pkgs { + fmt.Println(p.Tag()) + } } diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 7534ed4e7..7d8fc837e 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -64,10 +64,10 @@ type Pkg struct { git *git } -// NewFromCLI creates a Pkg from a set of CLI arguments. Calls fs.Parse() -func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { +// NewFromCLI creates a range of Pkg from a set of CLI arguments. Calls fs.Parse() +func NewFromCLI(fs *flag.FlagSet, args ...string) ([]Pkg, error) { // Defaults - pi := pkgInfo{ + piBase := pkgInfo{ Org: "linuxkit", Arches: []string{"amd64", "arm64", "s390x"}, GitRepo: "https://github.com/linuxkit/linuxkit", @@ -80,175 +80,193 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { // Ideally want to look at every directory from root to `pkg` // for this file but might be tricky to arrange ordering-wise. - // These override fields in pi below, bools are in both forms to allow user overrides in either direction - argDisableCache := fs.Bool("disable-cache", pi.DisableCache, "Disable build cache") - argEnableCache := fs.Bool("enable-cache", !pi.DisableCache, "Enable build cache") - argNoNetwork := fs.Bool("nonetwork", !pi.Network, "Disallow network use during build") - argNetwork := fs.Bool("network", pi.Network, "Allow network use during build") + // These override fields in pi below, bools are in both forms to allow user overrides in either direction. + // These will apply to all packages built. + argDisableCache := fs.Bool("disable-cache", piBase.DisableCache, "Disable build cache") + argEnableCache := fs.Bool("enable-cache", !piBase.DisableCache, "Enable build cache") + argNoNetwork := fs.Bool("nonetwork", !piBase.Network, "Disallow network use during build") + argNetwork := fs.Bool("network", piBase.Network, "Allow network use during build") - argOrg := fs.String("org", pi.Org, "Override the hub org") + argOrg := fs.String("org", piBase.Org, "Override the hub org") // Other arguments var buildYML, hash, hashCommit, hashPath string var dirty, devMode bool + fs.StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file") fs.StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)") fs.StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash") fs.StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)") - fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg to be considered dirty") + fs.BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty") fs.BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively") - fs.Parse(args) + _ = fs.Parse(args) if fs.NArg() < 1 { - return Pkg{}, fmt.Errorf("A pkg directory is required") - } - if fs.NArg() > 1 { - return Pkg{}, fmt.Errorf("Unknown extra arguments given: %s", fs.Args()[1:]) + return nil, fmt.Errorf("At least one pkg directory is required") } - pkg := fs.Arg(0) - pkgPath, err := filepath.Abs(pkg) - if err != nil { - return Pkg{}, err - } - - if hashPath == "" { - hashPath = pkgPath - } else { - hashPath, err = filepath.Abs(hashPath) + var pkgs []Pkg + for _, pkg := range fs.Args() { + var ( + pkgHashPath string + pkgHash = hash + ) + pkgPath, err := filepath.Abs(pkg) if err != nil { - return Pkg{}, err + return nil, err } - if !strings.HasPrefix(pkgPath, hashPath) { - return Pkg{}, fmt.Errorf("Hash path is not a prefix of the package path") - } - - // TODO(ijc) pkgPath and hashPath really ought to be in the same git tree too... - } - - b, err := ioutil.ReadFile(filepath.Join(pkgPath, buildYML)) - if err != nil { - return Pkg{}, err - } - if err := yaml.Unmarshal(b, &pi); err != nil { - return Pkg{}, err - } - - if pi.Image == "" { - return Pkg{}, fmt.Errorf("Image field is required") - } - - dockerDepends, err := newDockerDepends(pkgPath, &pi) - if err != nil { - return Pkg{}, err - } - - if devMode { - // If --org is also used then this will be overwritten - // by argOrg when we iterate over the provided options - // in the fs.Visit block below. - pi.Org = os.Getenv("USER") - if hash == "" { - hash = "dev" - } - } - - // Go's flag package provides no way to see if a flag was set - // apart from Visit which iterates over only those which were - // set. - fs.Visit(func(f *flag.Flag) { - switch f.Name { - case "disable-cache": - pi.DisableCache = *argDisableCache - case "enable-cache": - pi.DisableCache = !*argEnableCache - case "network": - pi.Network = *argNetwork - case "nonetwork": - pi.Network = !*argNoNetwork - case "org": - pi.Org = *argOrg - } - }) - - var srcHashes string - sources := []pkgSource{{src: pkgPath, dst: "/"}} - - for _, source := range pi.ExtraSources { - tmp := strings.Split(source, ":") - if len(tmp) != 2 { - return Pkg{}, fmt.Errorf("Bad source format in %s", source) - } - srcPath := filepath.Clean(tmp[0]) // Should work with windows paths - dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path - - if !filepath.IsAbs(srcPath) { - srcPath = filepath.Join(pkgPath, srcPath) - } - - g, err := newGit(srcPath) - if err != nil { - return Pkg{}, err - } - if g == nil { - return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath) - } - h, err := g.treeHash(srcPath, hashCommit) - if err != nil { - return Pkg{}, err - } - - srcHashes += h - sources = append(sources, pkgSource{src: srcPath, dst: dstPath}) - } - - git, err := newGit(pkgPath) - if err != nil { - return Pkg{}, err - } - - if git != nil { - gitDirty, err := git.isDirty(hashPath, hashCommit) - if err != nil { - return Pkg{}, err - } - - dirty = dirty || gitDirty - - if hash == "" { - if hash, err = git.treeHash(hashPath, hashCommit); err != nil { - return Pkg{}, err + if hashPath == "" { + pkgHashPath = pkgPath + } else { + pkgHashPath, err = filepath.Abs(hashPath) + if err != nil { + return nil, err } - if srcHashes != "" { - hash += srcHashes - hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash))) + if !strings.HasPrefix(pkgPath, pkgHashPath) { + return nil, fmt.Errorf("Hash path is not a prefix of the package path") } - if dirty { - hash += "-dirty" + // TODO(ijc) pkgPath and hashPath really ought to be in the same git tree too... + } + + // make our own copy of piBase. We could use some deepcopy library, but it is just as easy to marshal/unmarshal + pib, err := yaml.Marshal(&piBase) + if err != nil { + return nil, err + } + var pi pkgInfo + if err := yaml.Unmarshal(pib, &pi); err != nil { + return nil, err + } + + b, err := ioutil.ReadFile(filepath.Join(pkgPath, buildYML)) + if err != nil { + return nil, err + } + + if err := yaml.Unmarshal(b, &pi); err != nil { + return nil, err + } + + if pi.Image == "" { + return nil, fmt.Errorf("Image field is required") + } + + dockerDepends, err := newDockerDepends(pkgPath, &pi) + if err != nil { + return nil, err + } + + if devMode { + // If --org is also used then this will be overwritten + // by argOrg when we iterate over the provided options + // in the fs.Visit block below. + pi.Org = os.Getenv("USER") + if pkgHash == "" { + pkgHash = "dev" } } - } - return Pkg{ - image: pi.Image, - org: pi.Org, - hash: hash, - commitHash: hashCommit, - arches: pi.Arches, - sources: sources, - gitRepo: pi.GitRepo, - network: pi.Network, - cache: !pi.DisableCache, - config: pi.Config, - dockerDepends: dockerDepends, - dirty: dirty, - path: pkgPath, - git: git, - }, nil + // Go's flag package provides no way to see if a flag was set + // apart from Visit which iterates over only those which were + // set. This must be run here, rather than earlier, because we need to + // have read it from the build.yml file first, then override based on CLI. + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "disable-cache": + pi.DisableCache = *argDisableCache + case "enable-cache": + pi.DisableCache = !*argEnableCache + case "network": + pi.Network = *argNetwork + case "nonetwork": + pi.Network = !*argNoNetwork + case "org": + pi.Org = *argOrg + } + }) + + var srcHashes string + sources := []pkgSource{{src: pkgPath, dst: "/"}} + + for _, source := range pi.ExtraSources { + tmp := strings.Split(source, ":") + if len(tmp) != 2 { + return nil, fmt.Errorf("Bad source format in %s", source) + } + srcPath := filepath.Clean(tmp[0]) // Should work with windows paths + dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path + + if !filepath.IsAbs(srcPath) { + srcPath = filepath.Join(pkgPath, srcPath) + } + + g, err := newGit(srcPath) + if err != nil { + return nil, err + } + if g == nil { + return nil, fmt.Errorf("Source %s not in a git repository", srcPath) + } + h, err := g.treeHash(srcPath, hashCommit) + if err != nil { + return nil, err + } + + srcHashes += h + sources = append(sources, pkgSource{src: srcPath, dst: dstPath}) + } + + git, err := newGit(pkgPath) + if err != nil { + return nil, err + } + + if git != nil { + gitDirty, err := git.isDirty(pkgHashPath, hashCommit) + if err != nil { + return nil, err + } + + dirty = dirty || gitDirty + + if pkgHash == "" { + if pkgHash, err = git.treeHash(pkgHashPath, hashCommit); err != nil { + return nil, err + } + + if srcHashes != "" { + pkgHash += srcHashes + pkgHash = fmt.Sprintf("%x", sha1.Sum([]byte(pkgHash))) + } + + if dirty { + pkgHash += "-dirty" + } + } + } + + pkgs = append(pkgs, Pkg{ + image: pi.Image, + org: pi.Org, + hash: pkgHash, + commitHash: hashCommit, + arches: pi.Arches, + sources: sources, + gitRepo: pi.GitRepo, + network: pi.Network, + cache: !pi.DisableCache, + config: pi.Config, + dockerDepends: dockerDepends, + dirty: dirty, + path: pkgPath, + git: git, + }) + } + return pkgs, nil } // Hash returns the hash of the package diff --git a/src/cmd/linuxkit/pkglib/pkglib_test.go b/src/cmd/linuxkit/pkglib/pkglib_test.go index 1fac3012b..de4b46c51 100644 --- a/src/cmd/linuxkit/pkglib/pkglib_test.go +++ b/src/cmd/linuxkit/pkglib/pkglib_test.go @@ -39,8 +39,9 @@ func testBool(t *testing.T, key string, inv bool, forceOn, forceOff string, get args = append(args, override) } args = append(args, pkgDir) - pkg, err := NewFromCLI(flags, args...) + pkgs, err := NewFromCLI(flags, args...) require.NoError(t, err) + pkg := pkgs[0] t.Logf("override %q produced %t", override, get(pkg)) f(t, pkg) }