add support for dynamically calculated build arg sets (#4156)

docs/packages.md

@@ -403,7 +403,7 @@ linuxkit has support for certain calculated build args for the value of the arg.
 
 All calculated build args are prefixed with `@lkt:`.
 
-* `@lkt:pkg:<path>` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag <path>`. The `<path>` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file <file>`, it is relative to the directory in which <file> exists.
+* `VAR=@lkt:pkg:<path>` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag <path>`. The `<path>` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file <file>`, it is relative to the directory in which <file> exists.
 
 For example:
 
@@ -413,6 +413,27 @@ buildArgs:
   - REL_HASH=@lkt:pkg:foo # will be replaced with the value of `linuxkit pkg show-tag foo` relative to this build.yml file
 ```
 
+* `VAR_%=@lkt:pkgs:<paths>` - (note `pkgs` plural) the linuxkit package hashes of the multiple packages satisfied by `<paths>`. linuxkit will get the linuxkit package hash of each path in `<paths>`, as determined by `linuxkit pkg show-tag <path>`. The `<paths>` can be absolute, or if provided as a relative path, it is relative to the working directory of the file which contains the build arg, whether `build.yml` in a package or the build arg
+file provided to `--build-arg-file <file>`. The `<paths>` supports basic shell globbing, such as `./foo/*` or `/var/foo{1,2,3}`. Globs that start with `.` will be ignored, e.g. `foo/*` will match `foo/one` and `foo/two` but not `foo/.git` and `foo/.bar`. For each package in `<paths>`, it will create a build arg with the name `VAR_<package-name>` and the value of the package hash, where: the `%` is replaced with the name of the package; an all `/` and `-` characters are replaced with `_`; all characters are upper-cased.
+
+There _must_ be at least one valid environment variable character before the `%` character.
+
+For example:
+
+```yaml
+buildArgs:
+  - DEP_HASH_%=@lkt:pkgs:/usr/local/foo/* 
+```
+
+If there are packages in `/usr/local/foo/` named `bar`, `baz`, and `qux`, and each of them has a package as shown
+by `linuxkit pkg show-tag` as `linuxkit/bar:123abc`, `linuxkit/baz:aabb666`, and `linuxkit/qux:bbcc777`, this will create the following build args:
+
+```
+DEP_HASH_LINUXKIT_BAR=linuxkit/bar:123abc
+DEP_HASH_LINUXKIT_BAZ=linuxkit/baz:aabb666
+DEP_HASH_LINUXKIT_QUX=linuxkit/qux:bbcc777
+```
+
 ## Releases
 
 Normally, whenever a package is updated, CI will build and push the package to Docker Hub by calling `linuxkit pkg push`.

src/cmd/linuxkit/pkg_build.go

@@ -129,7 +129,7 @@ func pkgBuildCmd() *cobra.Command {
 						return fmt.Errorf("error transforming build arg %s: %v", line, err)
 					}
 
-					buildArgs = append(buildArgs, buildArg)
+					buildArgs = append(buildArgs, buildArg...)
 				}
 				if err := scanner.Err(); err != nil {
 					return fmt.Errorf("error reading build args file %s: %w", filename, err)
@@ -138,9 +138,9 @@ func pkgBuildCmd() *cobra.Command {
 			opts = append(opts, pkglib.WithBuildArgs(buildArgs))
 
 			// also need to parse the build args from the build.yml file for any special linuxkit values
-			for _, p := range pkgs {
-				if err := p.ProcessBuildArgs(); err != nil {
-					return fmt.Errorf("error processing build args for package %q: %w", p.Tag(), err)
+			for i := range pkgs {
+				if err := pkgs[i].ProcessBuildArgs(); err != nil {
+					return fmt.Errorf("error processing build args for package %q: %w", pkgs[i].Tag(), err)
 				}
 			}
 

src/cmd/linuxkit/pkglib/buildarg.go

@@ -2,6 +2,7 @@ package pkglib
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 )
@@ -9,22 +10,24 @@ import (
 const (
 	buildArgSpecialPrefix = "@lkt:"
 	buildArgPkgPrefix     = "pkg:"
+	buildArgsPkgPrefix    = "pkgs:"
+	buildArgsKeyStemChar  = "%"
 )
 
 // TransformBuildArgValue transforms a build arg pair whose value starts with the special linuxkit prefix.
-func TransformBuildArgValue(line, anchorFile string) (string, error) {
+func TransformBuildArgValue(line, anchorFile string) ([]string, error) {
 	parts := strings.SplitN(line, "=", 2)
 	if len(parts) != 2 {
-		return "", fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line)
+		return nil, fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line)
 	}
 	key := parts[0]
 	val := parts[1]
 	// check if the value is a special linuxkit value
 	if !strings.HasPrefix(val, buildArgSpecialPrefix) {
-		return line, nil
+		return []string{line}, nil
 	}
 	stripped := strings.TrimPrefix(val, buildArgSpecialPrefix)
-	var final string
+	var final []string
 	// see if we know what kind of value it is
 	switch {
 	case strings.HasPrefix(stripped, buildArgPkgPrefix):
@@ -33,23 +36,74 @@ func TransformBuildArgValue(line, anchorFile string) (string, error) {
 		if !strings.HasPrefix(pkgPath, "/") {
 			anchorDir, err := filepath.Abs(filepath.Dir(anchorFile))
 			if err != nil {
-				return "", fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err)
+				return nil, fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err)
 			}
 			pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath))
 		}
 		pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, pkgPath)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 		if len(pkgs) == 0 {
-			return "", fmt.Errorf("no package found at path %q", pkgPath)
+			return nil, fmt.Errorf("no package found at path %q", pkgPath)
 		}
 		p := pkgs[0]
 		tag := p.Tag()
-		final = tag
+		final = append(final, fmt.Sprintf("%s=%s", key, tag))
+	case strings.HasPrefix(stripped, buildArgsPkgPrefix):
+		// validate the key
+		if !strings.Contains(key, buildArgsKeyStemChar) {
+			return nil, fmt.Errorf("invalid build arg key %q, must contain a '%s'", key, buildArgsKeyStemChar)
+		}
+		pkgPath := strings.TrimPrefix(stripped, buildArgsPkgPrefix)
+		if !strings.HasPrefix(pkgPath, "/") {
+			anchorDir, err := filepath.Abs(filepath.Dir(anchorFile))
+			if err != nil {
+				return nil, fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err)
+			}
+			pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath))
+		}
+		matches, err := filepath.Glob(pkgPath)
+		if err != nil {
+			return nil, fmt.Errorf("error globbing package path %q: %v", pkgPath, err)
+		}
+		if len(matches) == 0 {
+			return nil, fmt.Errorf("no packages found matching path %q", pkgPath)
+		}
+		var finalMatches []string
+		for _, match := range matches {
+			// ensure the match is a directory
+			info, err := os.Stat(match)
+			if err != nil {
+				return nil, fmt.Errorf("error stating package path %q: %v", match, err)
+			}
+			if !info.IsDir() {
+				continue
+			}
+			if strings.HasPrefix(info.Name(), ".") {
+				continue
+			}
+			finalMatches = append(finalMatches, match)
+		}
+		pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, finalMatches...)
+		if err != nil {
+			return nil, fmt.Errorf("error loading packages from paths %q: %v", pkgPath, err)
+		}
+		if len(pkgs) == 0 {
+			return nil, fmt.Errorf("no packages found at path %q", pkgPath)
+		}
+		for _, p := range pkgs {
+			tag := p.Tag()
+			// generate the special build arg key
+			image := strings.ReplaceAll(p.Image(), "/", "_")
+			image = strings.ReplaceAll(image, "-", "_")
+			image = strings.ToUpper(image)
+			updatedKey := strings.ReplaceAll(key, buildArgsKeyStemChar, image)
+			final = append(final, fmt.Sprintf("%s=%s", updatedKey, tag))
+		}
 	default:
 		// something unknown
-		return "", fmt.Errorf("unknown linuxkit build arg value %q", val)
+		return nil, fmt.Errorf("unknown linuxkit build arg value %q", val)
 	}
-	return fmt.Sprintf("%s=%s", key, final), nil
+	return final, nil
 }

src/cmd/linuxkit/pkglib/pkglib.go

@@ -366,16 +366,21 @@ func (p Pkg) cleanForBuild() error {
 	return nil
 }
 
-func (p Pkg) ProcessBuildArgs() error {
+func (p *Pkg) ProcessBuildArgs() error {
 	if p.buildArgs == nil {
 		return nil
 	}
-	var err error
-	for i, arg := range *p.buildArgs {
-		(*p.buildArgs)[i], err = TransformBuildArgValue(arg, p.buildYML)
+	var buildArgs []string
+	for _, arg := range *p.buildArgs {
+		transformedLine, err := TransformBuildArgValue(arg, p.buildYML)
 		if err != nil {
 			return fmt.Errorf("error processing build arg %q: %v", arg, err)
 		}
+		buildArgs = append(buildArgs, transformedLine...)
+	}
+	// Replace the original build args with the transformed ones
+	if len(buildArgs) > 0 {
+		p.buildArgs = &buildArgs
 	}
 	return nil
 }

test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh

@@ -19,10 +19,6 @@ clean_up() {
 }
 trap clean_up EXIT
 
-# to be clear
-pwd
-ls -la .
-
 for i in "${TMPDIR1}" "${TMPDIR2}"; do
     rm -rf "${i}"
     mkdir -p "${i}"
@@ -57,7 +53,6 @@ current=$(linuxkit pkg show-tag .)
 # dump it to a filesystem
 linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf -
 # for extra debugging
-find "${TMPEXPORT}" -type f -exec ls -la {} \;
 actual1=$(cat ${TMPEXPORT}/var/hash1)
 actual2=$(cat ${TMPEXPORT}/var/hash2)
 

test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh

@@ -19,10 +19,6 @@ clean_up() {
 }
 trap clean_up EXIT
 
-# to be clear
-pwd
-ls -la .
-
 for i in "${TMPDIR1}" "${TMPDIR2}"; do
     rm -rf "${i}"
     mkdir -p "${i}"
@@ -57,7 +53,6 @@ current=$(linuxkit pkg show-tag .)
 # dump it to a filesystem
 linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf -
 # for extra debugging
-find "${TMPEXPORT}" -type f -exec ls -la {} \;
 actual1=$(cat ${TMPEXPORT}/var/hash1)
 actual2=$(cat ${TMPEXPORT}/var/hash2)
 

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/.gitignore

@@ -0,0 +1 @@
+foo/

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.21
+ARG HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_ONE
+ARG HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_TWO
+RUN printf '%s\n' "${HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_ONE}" > /var/hash1
+RUN printf '%s\n' "${HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_TWO}" > /var/hash2

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-args

@@ -0,0 +1 @@
+HASH_%=@lkt:pkgs:/tmp/foo/*

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-file.yml

@@ -0,0 +1,2 @@
+org: linuxkit
+image: hashes-in-build-args-file

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build.yml

@@ -0,0 +1,4 @@
+org: linuxkit
+image: hashes-in-build-args-yaml
+buildArgs:
+- HASH_%=@lkt:pkgs:/tmp/foo/*

test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/test.sh

@@ -0,0 +1,94 @@
+#!/bin/sh
+# SUMMARY: Check that tar output format build is reproducible
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+# need to build the special dir /tmp/bar12345 first
+TMPDIR=/tmp/foo
+TMPDIR1=${TMPDIR}/one
+TMPDIR2=${TMPDIR}/two
+TMPEXPORT=$(mktemp -d)
+CACHE_DIR=$(mktemp -d)
+
+clean_up() {
+	rm -rf ${TMPDIR} ${CACHE_DIR} ${TMPEXPORT}
+}
+trap clean_up EXIT
+
+for i in "${TMPDIR1}" "${TMPDIR2}"; do
+    rm -rf "${i}"
+    mkdir -p "${i}"
+    echo "This is a test file for the special build arg" > "${i}/test"
+    cat > "${i}/build.yml" <<EOF
+org: linuxkit
+image: hashes-in-build-args-$(basename "${i}")
+EOF
+done
+git -C "${TMPDIR}" init
+git -C "${TMPDIR}" config user.email "you@example.com"
+git -C "${TMPDIR}" config user.name "Your Name"
+git -C "${TMPDIR}" add .
+git -C "${TMPDIR}" commit -m "Initial commit for special build arg test"
+
+expected1=$(linuxkit pkg show-tag "${TMPDIR1}")
+expected2=$(linuxkit pkg show-tag "${TMPDIR2}")
+
+# print it out for the logs
+echo "Building packages with special build args from ${TMPDIR1} and ${TMPDIR2}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR1}"
+linuxkit --cache ${CACHE_DIR} pkg show-tag "${TMPDIR2}"
+
+## Run two tests: with build args in yaml and with build arg file
+targetarch="arm64"
+for yml in build.yml build-file.yml; do
+    extra_args=""
+    if [ "$yml" = "build-file.yml" ]; then
+        extra_args="--build-arg-file ./build-args"
+    fi
+    linuxkit --cache ${CACHE_DIR} pkg build --platforms linux/${targetarch} --force --build-yml "${yml}" . ${extra_args} 2>&1
+    if [ $? -ne 0 ]; then
+        echo "Build failed"
+        exit 1
+    fi
+
+    current=$(linuxkit pkg show-tag --build-yml "${yml}" .)
+
+    # for debugging
+    find ${CACHE_DIR} -ls
+    cat ${CACHE_DIR}/index.json
+    index=$(cat ${CACHE_DIR}/index.json | jq -r '.manifests[] | select(.annotations["org.opencontainers.image.ref.name"] == "'${current}'") | .digest' | cut -d: -f2)
+    echo "Current package index: ${index}"
+    cat ${CACHE_DIR}/blobs/sha256/${index} | jq '.'
+    manifest=$(cat ${CACHE_DIR}/blobs/sha256/${index} | jq -r '.manifests[] | select(.platform.architecture == "'${targetarch}'") | .digest' | cut -d: -f2)
+    echo "Current package manifest: ${manifest}"
+    cat ${CACHE_DIR}/blobs/sha256/${manifest} | jq '.'
+
+
+    # dump it to a filesystem
+    rm -rf "${TMPEXPORT}/*"
+    linuxkit --cache ${CACHE_DIR} cache export --platform linux/${targetarch} --format filesystem --outfile /tmp/lktout123.tar "${current}"
+    file /tmp/lktout123.tar
+    ls -l /tmp/lktout123.tar
+    cat /tmp/lktout123.tar | tar -C "${TMPEXPORT}" -xvf -
+    # for extra debugging
+    actual1=$(cat ${TMPEXPORT}/var/hash1)
+    actual2=$(cat ${TMPEXPORT}/var/hash2)
+
+
+    if [ "${expected1}" != "${actual1}" ]; then
+        echo "Expected HASH1: ${expected1}, but got: ${actual1}"
+        exit 1
+    fi
+
+    if [ "${expected2}" != "${actual2}" ]; then
+        echo "Expected HASH2: ${expected2}, but got: ${actual2}"
+        exit 1
+    fi
+done
+
+exit 0