Merge pull request #3986 from deitch/dockerfile-in-build-yml

Dockerfile in build yml and CLI; tag templates

docs/packages.md

@@ -50,6 +50,7 @@ A package source consists of a directory containing at least two files:
 
 - `image` _(string)_: *(mandatory)* The name of the image to build
 - `org` _(string)_: The hub/registry organisation to which this package belongs
+- `dockerfile` _(string)_: The dockerfile to use to build this package, must be in this directory or below (default: `Dockerfile`)
 - `arches` _(list of string)_: The architectures which this package should be built for (valid entries are `GOARCH` names)
 - `extra-sources` _(list of strings)_: Additional sources for the package outside the package directory. The format is `src:dst`, where `src` can be relative to the package directory and `dst` is the destination in the build context. This is useful for sharing files, such as vendored go code, between packages.
 - `gitrepo` _(string)_: The git repository where the package source is kept.

src/cmd/linuxkit/pkg.go

@@ -22,6 +22,7 @@ func pkgCmd() *cobra.Command {
 		hashPath        string
 		dirty           bool
 		devMode         bool
+		tag             string
 	)
 
 	cmd := &cobra.Command{
@@ -36,6 +37,7 @@ func pkgCmd() *cobra.Command {
 				HashPath:   hashPath,
 				Dirty:      dirty,
 				Dev:        devMode,
+				Tag:        tag,
 			}
 			if cmd.Flags().Changed("disable-cache") && cmd.Flags().Changed("enable-cache") {
 				return errors.New("cannot set but disable-cache and enable-cache")
@@ -85,6 +87,7 @@ func pkgCmd() *cobra.Command {
 	cmd.PersistentFlags().StringVar(&argOrg, "org", piBase.Org, "Override the hub org")
 	cmd.PersistentFlags().StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file")
 	cmd.PersistentFlags().StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)")
+	cmd.PersistentFlags().StringVar(&tag, "tag", "{{.Hash}}", "Override the tag using fixed strings and/or text templates. Acceptable are .Hash for the hash")
 	cmd.PersistentFlags().StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash")
 	cmd.PersistentFlags().StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)")
 	cmd.PersistentFlags().BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty")

src/cmd/linuxkit/pkg_build.go

@@ -45,6 +45,7 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
 		manifest       bool
 		cacheDir       = flagOverEnvVarOverDefaultString{def: defaultLinuxkitCache(), envVar: envVarCacheDir}
 		sbomScanner    string
+		dockerfile     string
 	)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
@@ -92,6 +93,7 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
 		if sbomScanner != "false" {
 			opts = append(opts, pkglib.WithBuildSbomScanner(sbomScanner))
 		}
+		opts = append(opts, pkglib.WithDockerfile(dockerfile))
 
 		// skipPlatformsMap contains platforms that should be skipped
 		skipPlatformsMap := make(map[string]bool)
@@ -128,12 +130,12 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
 		// look for builders env var
 		buildersMap, err = buildPlatformBuildersMap(os.Getenv(buildersEnvVar), buildersMap)
 		if err != nil {
-			return fmt.Errorf("error in environment variable %s: %w\n", buildersEnvVar, err)
+			return fmt.Errorf("error in environment variable %s: %w", buildersEnvVar, err)
 		}
 		// any CLI options override env var
 		buildersMap, err = buildPlatformBuildersMap(builders, buildersMap)
 		if err != nil {
-			return fmt.Errorf("error in --builders flag: %w\n", err)
+			return fmt.Errorf("error in --builders flag: %w", err)
 		}
 		opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
 		opts = append(opts, pkglib.WithBuildBuilderImage(builderImage))
@@ -202,6 +204,7 @@ func addCmdRunPkgBuildPush(cmd *cobra.Command, withPush bool) *cobra.Command {
 	cmd.Flags().BoolVar(&nobuild, "nobuild", false, "Skip building the image before pushing, conflicts with -force")
 	cmd.Flags().BoolVar(&manifest, "manifest", true, "Create and push multi-arch manifest")
 	cmd.Flags().StringVar(&sbomScanner, "sbom-scanner", "", "SBOM scanner to use, must match the buildkit spec; set to blank to use the buildkit default; set to 'false' for no scanning")
+	cmd.Flags().StringVar(&dockerfile, "dockerfile", "", "Dockerfile to use for building the image, must be in this directory or below, overrides what is in build.yml")
 
 	return cmd
 }

src/cmd/linuxkit/pkglib/build.go

@@ -43,6 +43,7 @@ type buildOpts struct {
 	builderRestart   bool
 	sbomScan         bool
 	sbomScannerImage string
+	dockerfile       string
 }
 
 // BuildOpt allows callers to specify options to Build
@@ -186,6 +187,14 @@ func WithBuildSbomScanner(scanner string) BuildOpt {
 	}
 }
 
+// WithDockerfile which dockerfile to use when building the package
+func WithDockerfile(dockerfile string) BuildOpt {
+	return func(bo *buildOpts) error {
+		bo.dockerfile = dockerfile
+		return nil
+	}
+}
+
 // Build builds the package
 func (p Pkg) Build(bos ...BuildOpt) error {
 	var bo buildOpts
@@ -211,6 +220,22 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 		return err
 	}
 
+	// validate the Dockerfile before bothing to move ahead, because this func call is public, so someone could
+	// pass something to it as a library call. We also check in the build function, to avoid multiple loops each with an error.
+
+	// if the dockerfile override was not set in the build options, i.e. it is empty, use the one from the package,
+	// which never should be empty. We set it onto the buildOpts, because that is what we use to pass it around to lower-level
+	// funcs.
+	if bo.dockerfile == "" {
+		bo.dockerfile = p.dockerfile
+	}
+	if strings.Contains(bo.dockerfile, "..") {
+		return fmt.Errorf("cannot expand beyond root of context for dockerfile %s", bo.dockerfile)
+	}
+	if _, err := os.Stat(filepath.Join(p.path, bo.dockerfile)); err != nil {
+		return fmt.Errorf("dockerfile %s does not exist or cannot be read in context %s", bo.dockerfile, p.path)
+	}
+
 	// did we have the build cache dir provided?
 	if bo.cacheDir == "" {
 		return errors.New("must provide linuxkit build cache directory")
@@ -593,7 +618,8 @@ func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c lktspec.CacheProvi
 	if bo.ignoreCache {
 		passCache = nil
 	}
-	if err := d.build(ctx, tagArch, p.path, builderName, builderImage, platform, restart, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, imageBuildOpts); err != nil {
+
+	if err := d.build(ctx, tagArch, p.path, bo.dockerfile, builderName, builderImage, platform, restart, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, imageBuildOpts); err != nil {
 		stdoutCloser()
 		if strings.Contains(err.Error(), "executor failed running [/dev/.buildkit_qemu_emulator") {
 			return nil, fmt.Errorf("buildkit was unable to emulate %s. check binfmt has been set up and works for this platform: %v", platform, err)

src/cmd/linuxkit/pkglib/build_test.go

@@ -58,7 +58,7 @@ func (d *dockerMocker) contextSupportCheck() error {
 func (d *dockerMocker) builder(_ context.Context, _, _, _ string, _ bool) (*buildkitClient.Client, error) {
 	return nil, fmt.Errorf("not implemented")
 }
-func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, builderRestart bool, c lktspec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts dockertypes.ImageBuildOptions) error {
+func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerfile, dockerContext, builderImage, platform string, builderRestart bool, c lktspec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts dockertypes.ImageBuildOptions) error {
 	if !d.enableBuild {
 		return errors.New("build disabled")
 	}
@@ -534,6 +534,7 @@ func TestBuild(t *testing.T) {
 				}
 				opts = append(opts, WithBuildPlatforms(targets...))
 			}
+			tt.p.dockerfile = "testdata/Dockerfile"
 			err := tt.p.Build(opts...)
 			switch {
 			case (tt.err == "" && err != nil) || (tt.err != "" && err == nil) || (tt.err != "" && err != nil && !strings.HasPrefix(err.Error(), tt.err)):

src/cmd/linuxkit/pkglib/docker.go

@@ -53,7 +53,7 @@ const (
 
 type dockerRunner interface {
 	tag(ref, tag string) error
-	build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts types.ImageBuildOptions) error
+	build(ctx context.Context, tag, pkg, dockerfile, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts types.ImageBuildOptions) error
 	save(tgt string, refs ...string) error
 	load(src io.Reader) error
 	pull(img string) (bool, error)
@@ -402,7 +402,7 @@ func (dr *dockerRunnerImpl) tag(ref, tag string) error {
 	return dr.command(nil, nil, nil, "image", "tag", ref, tag)
 }
 
-func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts types.ImageBuildOptions) error {
+func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerfile, dockerContext, builderImage, platform string, restart bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage string, imageBuildOpts types.ImageBuildOptions) error {
 	// ensure we have a builder
 	client, err := dr.builder(ctx, dockerContext, builderImage, platform, restart)
 	if err != nil {
@@ -473,26 +473,26 @@ func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext,
 		solveOpts.Session = append(solveOpts.Session, up)
 	} else {
 		solveOpts.LocalDirs = map[string]string{
-			builder.DefaultLocalNameDockerfile: pkg,
+			builder.DefaultLocalNameDockerfile: path.Join(pkg, dockerfile),
 			builder.DefaultLocalNameContext:    pkg,
 		}
 	}
 
 	// go through the dockerfile to see if we have any provided images cached
 	if c != nil {
-		dockerfile := path.Join(pkg, "Dockerfile")
-		f, err := os.Open(dockerfile)
+		dockerfileRef := path.Join(pkg, dockerfile)
+		f, err := os.Open(dockerfileRef)
 		if err != nil {
-			return fmt.Errorf("error opening dockerfile %s: %v", dockerfile, err)
+			return fmt.Errorf("error opening dockerfile %s: %v", dockerfileRef, err)
 		}
 		defer f.Close()
 		ast, err := parser.Parse(f)
 		if err != nil {
-			return fmt.Errorf("error parsing dockerfile from bytes into AST %s: %v", dockerfile, err)
+			return fmt.Errorf("error parsing dockerfile from bytes into AST %s: %v", dockerfileRef, err)
 		}
 		stages, metaArgs, err := instructions.Parse(ast.AST)
 		if err != nil {
-			return fmt.Errorf("error parsing dockerfile from AST into stages %s: %v", dockerfile, err)
+			return fmt.Errorf("error parsing dockerfile from AST into stages %s: %v", dockerfileRef, err)
 		}
 
 		// fill optMetaArgs with args found while parsing Dockerfile

src/cmd/linuxkit/pkglib/pkglib.go

@@ -1,12 +1,14 @@
 package pkglib
 
 import (
+	"bytes"
 	"crypto/sha1"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"text/template"
 
 	"gopkg.in/yaml.v2"
 
@@ -18,6 +20,7 @@ import (
 type pkgInfo struct {
 	Image        string            `yaml:"image"`
 	Org          string            `yaml:"org"`
+	Dockerfile   string            `yaml:"dockerfile"`
 	Arches       []string          `yaml:"arches"`
 	ExtraSources []string          `yaml:"extra-sources"`
 	GitRepo      string            `yaml:"gitrepo"` // ??
@@ -49,8 +52,10 @@ type PkglibConfig struct {
 	HashPath     string
 	Dirty        bool
 	Dev          bool
+	Tag          string // Tag is a text/template string, defaults to {{.Hash}}
 }
 
+// NewPkInfo returns a new pkgInfo with default values
 func NewPkgInfo() pkgInfo {
 	return pkgInfo{
 		Org:          "linuxkit",
@@ -58,6 +63,7 @@ func NewPkgInfo() pkgInfo {
 		GitRepo:      "https://github.com/linuxkit/linuxkit",
 		Network:      false,
 		DisableCache: false,
+		Dockerfile:   "Dockerfile",
 	}
 }
 
@@ -84,7 +90,9 @@ type Pkg struct {
 
 	// Internal state
 	path       string
+	dockerfile string
 	hash       string
+	tag        string
 	dirty      bool
 	commitHash string
 	git        *git
@@ -250,6 +258,16 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) {
 			}
 		}
 
+		// calculate the tag to use based on the template and the pkgHash
+		tmpl, err := template.New("tag").Parse(cfg.Tag)
+		if err != nil {
+			return nil, fmt.Errorf("invalid tag template: %v", err)
+		}
+		var buf bytes.Buffer
+		if err := tmpl.Execute(&buf, map[string]string{"Hash": pkgHash}); err != nil {
+			return nil, fmt.Errorf("failed to execute tag template: %v", err)
+		}
+		tag := buf.String()
 		pkgs = append(pkgs, Pkg{
 			image:         pi.Image,
 			org:           pi.Org,
@@ -265,7 +283,9 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) {
 			dockerDepends: dockerDepends,
 			dirty:         dirty,
 			path:          pkgPath,
+			dockerfile:    pi.Dockerfile,
 			git:           git,
+			tag:           tag,
 		})
 	}
 	return pkgs, nil
@@ -290,7 +310,7 @@ func (p Pkg) ReleaseTag(release string) (string, error) {
 
 // Tag returns the tag to use for the package
 func (p Pkg) Tag() string {
-	t := p.hash
+	t := p.tag
 	if t == "" {
 		t = "latest"
 	}

test/cases/000_build/055_dockerfiles/000_missing_in_yml/Dockerfile

@@ -0,0 +1 @@
+FROM alpine:3.19

test/cases/000_build/055_dockerfiles/000_missing_in_yml/build.yml

@@ -0,0 +1,3 @@
+org: linuxkit
+image: missing-in-yml
+dockerfile: Dockerfilenotexist

test/cases/000_build/055_dockerfiles/000_missing_in_yml/test.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+set +e
+linuxkit pkg build --force .
+command_status=$?
+set -e
+
+if [ $command_status -eq 0 ]; then
+    echo "Command should have failed"
+    exit 1
+fi
+
+exit 0

test/cases/000_build/055_dockerfiles/001_missing_in_yml_found_cli/Dockerfile

@@ -0,0 +1 @@
+FROM alpine:3.19

test/cases/000_build/055_dockerfiles/001_missing_in_yml_found_cli/build.yml

@@ -0,0 +1,3 @@
+org: linuxkit
+image: missing-in-yml
+dockerfile: Dockerfilenotexist

test/cases/000_build/055_dockerfiles/001_missing_in_yml_found_cli/test.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+linuxkit pkg build --force --dockerfile Dockerfile .
+
+exit 0

test/cases/000_build/055_dockerfiles/002_found_in_yml_missing_cli/Dockerfile

@@ -0,0 +1 @@
+FROM alpine:3.19

test/cases/000_build/055_dockerfiles/002_found_in_yml_missing_cli/build.yml

@@ -0,0 +1,3 @@
+org: linuxkit
+image: missing-in-yml
+dockerfile: Dockerfile

test/cases/000_build/055_dockerfiles/002_found_in_yml_missing_cli/test.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+set +e
+linuxkit pkg build --force --dockerfile nosuchfile .
+command_status=$?
+set -e
+
+if [ $command_status -eq 0 ]; then
+    echo "Command should have failed"
+    exit 1
+fi
+
+exit 0

test/cases/000_build/055_dockerfiles/003_default/Dockerfile

@@ -0,0 +1 @@
+FROM alpine:3.19

test/cases/000_build/055_dockerfiles/003_default/build.yml

@@ -0,0 +1,2 @@
+org: linuxkit
+image: missing-in-yml

test/cases/000_build/055_dockerfiles/003_default/test.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+linuxkit pkg build --force .
+
+exit 0