add support for custom build args (#4155)

Signed-off-by: Avi Deitcher <avi@deitcher.net>

docs/packages.md

@@ -57,7 +57,7 @@ A package source consists of a directory containing at least two files:
 - `gitrepo` _(string)_: The git repository where the package source is kept.
 - `network` _(bool)_: Allow network access during the package build (default: no)
 - `disable-cache` _(bool)_: Disable build cache for this package (default: no)
-- `buildArgs` will forward a list of build arguments down to docker. As if `--build-arg` was specified during `docker build`
+- `buildArgs` will forward a list of build arguments down to docker. As if `--build-arg` was specified during `docker build`. See [BuildArgs][BuildArgs] for more information.
 - `config`: _(struct `github.com/moby/tool/src/moby.ImageConfig`)_: Image configuration, marshalled to JSON and added as `org.mobyproject.config` label on image (default: no label)
 - `depends`: Contains information on prerequisites which must be satisfied in order to build the package. Has subfields:
     - `docker-images`: Docker images to be made available (as `tar` files via `docker image save`) within the package build context. Contains the following nested fields:
@@ -382,6 +382,37 @@ ARG all_proxy
 LinuxKit does not judge between lower-cased or upper-cased variants of these options, e.g. `http_proxy` vs `HTTP_PROXY`,
 as `docker build` does not either. It just passes them through "as-is".
 
+## Build Args
+
+`linuxkit` does not support passing random CLI flags for build arguments when building packages.
+This is inline with its philosophy, of having as reproducible builds as possible, which requires
+everything to be available on disk and in the repository.
+
+It is possible to bypass this, but this is not recommended.
+
+As described in [Preset build arguments][Preset build arguments], linuxkit automatically sets some build arguments
+when building packages. However, you can also set your own build arguments, which will be passed to the
+`docker build` command.
+You can include your own build args in several ways.
+
+* `build.yml` - you can add a `buildArgs` field to the `build.yml` file, which will be passed as `--build-arg` to `docker build`.
+* `linuxkit pkg build` - you can pass the `--build-arg-file <file>` flag, with one `<key>=<value>` pair per line, which will be passed as `--build-arg` to `docker build`.
+
+When parsing for build args, whether from `build.yml`'s `buildArgs` field or from the `--build-arg-file`,
+linuxkit has support for certain calculated build args for the value of the arg. You can set these using the following syntax.
+
+All calculated build args are prefixed with `@lkt:`.
+
+* `@lkt:pkg:<path>` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag <path>`. The `<path>` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file <file>`, it is relative to the directory in which <file> exists.
+
+For example:
+
+```yaml
+buildArgs:
+  - DEP_HASH=@lkt:pkg:/usr/local/foo # will be replaced with the value of `linuxkit pkg show-tag /usr/local/foo`
+  - REL_HASH=@lkt:pkg:foo # will be replaced with the value of `linuxkit pkg show-tag foo` relative to this build.yml file
+```
+
 ## Releases
 
 Normally, whenever a package is updated, CI will build and push the package to Docker Hub by calling `linuxkit pkg push`.

src/cmd/linuxkit/const.go

@@ -1,6 +1,10 @@
 package main
 
-const (
+import (
-	defaultPkgBuildYML = "build.yml"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib"
-	defaultPkgCommit   = "HEAD"
+)
+
+const (
+	defaultPkgBuildYML = pkglib.DefaultPkgBuildYML
+	defaultPkgCommit   = pkglib.DefaultPkgCommit
 )

src/cmd/linuxkit/pkg_build.go

@@ -122,7 +122,14 @@ func pkgBuildCmd() *cobra.Command {
 				defer func() { _ = f.Close() }()
 				scanner := bufio.NewScanner(f)
 				for scanner.Scan() {
-					buildArgs = append(buildArgs, scanner.Text())
+					line := strings.TrimSpace(scanner.Text())
+					// check if the value is a special linuxkit value
+					buildArg, err := pkglib.TransformBuildArgValue(line, filename)
+					if err != nil {
+						return fmt.Errorf("error transforming build arg %s: %v", line, err)
+					}
+
+					buildArgs = append(buildArgs, buildArg)
 				}
 				if err := scanner.Err(); err != nil {
 					return fmt.Errorf("error reading build args file %s: %w", filename, err)
@@ -130,6 +137,13 @@ func pkgBuildCmd() *cobra.Command {
 			}
 			opts = append(opts, pkglib.WithBuildArgs(buildArgs))
 
+			// also need to parse the build args from the build.yml file for any special linuxkit values
+			for _, p := range pkgs {
+				if err := p.ProcessBuildArgs(); err != nil {
+					return fmt.Errorf("error processing build args for package %q: %w", p.Tag(), err)
+				}
+			}
+
 			// skipPlatformsMap contains platforms that should be skipped
 			skipPlatformsMap := make(map[string]bool)
 			if skipPlatforms != "" {

src/cmd/linuxkit/pkglib/buildarg.go

@@ -0,0 +1,55 @@
+package pkglib
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	buildArgSpecialPrefix = "@lkt:"
+	buildArgPkgPrefix     = "pkg:"
+)
+
+// TransformBuildArgValue transforms a build arg pair whose value starts with the special linuxkit prefix.
+func TransformBuildArgValue(line, anchorFile string) (string, error) {
+	parts := strings.SplitN(line, "=", 2)
+	if len(parts) != 2 {
+		return "", fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line)
+	}
+	key := parts[0]
+	val := parts[1]
+	// check if the value is a special linuxkit value
+	if !strings.HasPrefix(val, buildArgSpecialPrefix) {
+		return line, nil
+	}
+	stripped := strings.TrimPrefix(val, buildArgSpecialPrefix)
+	var final string
+	// see if we know what kind of value it is
+	switch {
+	case strings.HasPrefix(stripped, buildArgPkgPrefix):
+		pkgPath := strings.TrimPrefix(stripped, buildArgPkgPrefix)
+		// see if it is an absolute or relative path
+		if !strings.HasPrefix(pkgPath, "/") {
+			anchorDir, err := filepath.Abs(filepath.Dir(anchorFile))
+			if err != nil {
+				return "", fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err)
+			}
+			pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath))
+		}
+		pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, pkgPath)
+		if err != nil {
+			return "", err
+		}
+		if len(pkgs) == 0 {
+			return "", fmt.Errorf("no package found at path %q", pkgPath)
+		}
+		p := pkgs[0]
+		tag := p.Tag()
+		final = tag
+	default:
+		// something unknown
+		return "", fmt.Errorf("unknown linuxkit build arg value %q", val)
+	}
+	return fmt.Sprintf("%s=%s", key, final), nil
+}

src/cmd/linuxkit/pkglib/const.go

@@ -0,0 +1,8 @@
+package pkglib
+
+const (
+	// DefaultPkgBuildYML is the default name of the package build file
+	DefaultPkgBuildYML = "build.yml"
+	// DefaultPkgCommit is the default commit to use for packages
+	DefaultPkgCommit = "HEAD"
+)

src/cmd/linuxkit/pkglib/pkglib.go

@@ -92,6 +92,7 @@ type Pkg struct {
 
 	// Internal state
 	path       string
+	buildYML   string // full path to the build.yml file, not just relative to path
 	dockerfile string
 	hash       string
 	tag        string
@@ -150,7 +151,8 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) {
 			return nil, err
 		}
 
-		b, err := os.ReadFile(filepath.Join(pkgPath, cfg.BuildYML))
+		buildYmlFile := filepath.Join(pkgPath, cfg.BuildYML)
+		b, err := os.ReadFile(buildYmlFile)
 		if err != nil {
 			return nil, err
 		}
@@ -292,6 +294,7 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) {
 			dockerDepends: dockerDepends,
 			dirty:         dirty,
 			path:          pkgPath,
+			buildYML:      buildYmlFile,
 			dockerfile:    pi.Dockerfile,
 			git:           git,
 			tag:           tag,
@@ -363,6 +366,20 @@ func (p Pkg) cleanForBuild() error {
 	return nil
 }
 
+func (p Pkg) ProcessBuildArgs() error {
+	if p.buildArgs == nil {
+		return nil
+	}
+	var err error
+	for i, arg := range *p.buildArgs {
+		(*p.buildArgs)[i], err = TransformBuildArgValue(arg, p.buildYML)
+		if err != nil {
+			return fmt.Errorf("error processing build arg %q: %v", arg, err)
+		}
+	}
+	return nil
+}
+
 // Expands path from relative to abs against base, ensuring the result is within base, but is not base itself. Field is the fieldname, to be used for constructing the error.
 func makeAbsSubpath(field, base, path string) (string, error) {
 	if path == "" {

test/cases/000_build/056_build_args/005_build_arg_special_yaml/.gitignore

@@ -0,0 +1 @@
+foo/

test/cases/000_build/056_build_args/005_build_arg_special_yaml/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.21
+ARG HASH1
+ARG HASH2
+RUN printf '%s\n' "${HASH1}" > /var/hash1
+RUN printf '%s\n' "${HASH2}" > /var/hash2

test/cases/000_build/056_build_args/005_build_arg_special_yaml/build.yml

@@ -0,0 +1,5 @@
+org: linuxkit
+image: hashes-in-build-args
+buildArgs:
+- HASH1=@lkt:pkg:foo/
+- HASH2=@lkt:pkg:/tmp/bar12345/

test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+# need to build the special dir /tmp/bar12345 first
+TMPDIR1=/tmp/bar12345
+TMPDIR2=./foo/
+TMPEXPORT=$(mktemp -d)
+CACHE_DIR=$(mktemp -d)
+
+clean_up() {
+	rm -rf ${TMPDIR1} ${TMPDIR2} ${CACHE_DIR} ${TMPEXPORT}
+}
+trap clean_up EXIT
+
+# to be clear
+pwd
+ls -la .
+
+for i in "${TMPDIR1}" "${TMPDIR2}"; do
+    rm -rf "${i}"
+    mkdir -p "${i}"
+    echo "This is a test file for the special build arg" > "${i}/test"
+    cat > "${i}/build.yml" <<EOF
+org: linuxkit
+image: hashes-in-build-args
+EOF
+    git -C "${i}" init
+    git -C "${i}" config user.email "you@example.com"
+    git -C "${i}" config user.name "Your Name"
+    git -C "${i}" add .
+    git -C "${i}" commit -m "Initial commit for special build arg test"
+done
+
+# print it out for the logs
+echo "Building packages with special build args from ${TMPDIR1} and ${TMPDIR2}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR1}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR2}"
+
+logs=$(linuxkit --cache ${CACHE_DIR} pkg build --force . 2>&1)
+if [ $? -ne 0 ]; then
+    echo "Build failed with logs:"
+    echo "${logs}"
+    exit 1
+fi
+
+expected1=$(linuxkit pkg show-tag "${TMPDIR1}")
+expected2=$(linuxkit pkg show-tag "${TMPDIR2}")
+current=$(linuxkit pkg show-tag .)
+
+# dump it to a filesystem
+linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf -
+# for extra debugging
+find "${TMPEXPORT}" -type f -exec ls -la {} \;
+actual1=$(cat ${TMPEXPORT}/var/hash1)
+actual2=$(cat ${TMPEXPORT}/var/hash2)
+
+
+if [ "${expected1}" != "${actual1}" ]; then
+    echo "Expected HASH1: ${expected1}, but got: ${actual1}"
+    exit 1
+fi
+
+if [ "${expected2}" != "${actual2}" ]; then
+    echo "Expected HASH2: ${expected2}, but got: ${actual2}"
+    exit 1
+fi
+
+# Check that the build args were correctly transformed
+
+exit 0

test/cases/000_build/056_build_args/006_build_arg_special_cli/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.21
+ARG HASH1
+ARG HASH2
+RUN printf '%s\n' "${HASH1}" > /var/hash1
+RUN printf '%s\n' "${HASH2}" > /var/hash2

test/cases/000_build/056_build_args/006_build_arg_special_cli/build-args

@@ -0,0 +1,2 @@
+HASH1=@lkt:pkg:foo/
+HASH2=@lkt:pkg:/tmp/bar67890/

test/cases/000_build/056_build_args/006_build_arg_special_cli/build.yml

@@ -0,0 +1,2 @@
+org: linuxkit
+image: test-image-in-yaml-build-args

test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+# need to build the special dir /tmp/bar12345 first
+TMPDIR1=/tmp/bar67890
+TMPDIR2=./foo/
+TMPEXPORT=$(mktemp -d)
+CACHE_DIR=$(mktemp -d)
+
+clean_up() {
+	rm -rf ${TMPDIR1} ${TMPDIR2} ${CACHE_DIR} ${TMPEXPORT}
+}
+trap clean_up EXIT
+
+# to be clear
+pwd
+ls -la .
+
+for i in "${TMPDIR1}" "${TMPDIR2}"; do
+    rm -rf "${i}"
+    mkdir -p "${i}"
+    echo "This is a test file for the special build arg" > "${i}/test"
+    cat > "${i}/build.yml" <<EOF
+org: linuxkit
+image: hashes-in-build-args
+EOF
+    git -C "${i}" init
+    git -C "${i}" config user.email "you@example.com"
+    git -C "${i}" config user.name "Your Name"
+    git -C "${i}" add .
+    git -C "${i}" commit -m "Initial commit for special build arg test"
+done
+
+# print it out for the logs
+echo "Building packages with special build args from ${TMPDIR1} and ${TMPDIR2}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR1}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR2}"
+
+logs=$(linuxkit --cache ${CACHE_DIR} pkg build --force --build-arg-file build-args . 2>&1)
+if [ $? -ne 0 ]; then
+    echo "Build failed with logs:"
+    echo "${logs}"
+    exit 1
+fi
+
+expected1=$(linuxkit pkg show-tag "${TMPDIR1}")
+expected2=$(linuxkit pkg show-tag "${TMPDIR2}")
+current=$(linuxkit pkg show-tag .)
+
+# dump it to a filesystem
+linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf -
+# for extra debugging
+find "${TMPEXPORT}" -type f -exec ls -la {} \;
+actual1=$(cat ${TMPEXPORT}/var/hash1)
+actual2=$(cat ${TMPEXPORT}/var/hash2)
+
+if [ "${expected1}" != "${actual1}" ]; then
+    echo "Expected HASH1: ${expected1}, but got: ${actual1}"
+    exit 1
+fi
+
+if [ "${expected2}" != "${actual2}" ]; then
+    echo "Expected HASH2: ${expected2}, but got: ${actual2}"
+    exit 1
+fi
+
+# Check that the build args were correctly transformed
+
+exit 0