diff --git a/docs/packages.md b/docs/packages.md index 3b223c770..79ab56cff 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -403,7 +403,7 @@ linuxkit has support for certain calculated build args for the value of the arg. All calculated build args are prefixed with `@lkt:`. -* `@lkt:pkg:` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag `. The `` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file `, it is relative to the directory in which exists. +* `VAR=@lkt:pkg:` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag `. The `` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file `, it is relative to the directory in which exists. For example: @@ -413,6 +413,27 @@ buildArgs: - REL_HASH=@lkt:pkg:foo # will be replaced with the value of `linuxkit pkg show-tag foo` relative to this build.yml file ``` +* `VAR_%=@lkt:pkgs:` - (note `pkgs` plural) the linuxkit package hashes of the multiple packages satisfied by ``. linuxkit will get the linuxkit package hash of each path in ``, as determined by `linuxkit pkg show-tag `. The `` can be absolute, or if provided as a relative path, it is relative to the working directory of the file which contains the build arg, whether `build.yml` in a package or the build arg +file provided to `--build-arg-file `. The `` supports basic shell globbing, such as `./foo/*` or `/var/foo{1,2,3}`. Globs that start with `.` will be ignored, e.g. `foo/*` will match `foo/one` and `foo/two` but not `foo/.git` and `foo/.bar`. For each package in ``, it will create a build arg with the name `VAR_` and the value of the package hash, where: the `%` is replaced with the name of the package; an all `/` and `-` characters are replaced with `_`; all characters are upper-cased. + +There _must_ be at least one valid environment variable character before the `%` character. + +For example: + +```yaml +buildArgs: + - DEP_HASH_%=@lkt:pkgs:/usr/local/foo/* +``` + +If there are packages in `/usr/local/foo/` named `bar`, `baz`, and `qux`, and each of them has a package as shown +by `linuxkit pkg show-tag` as `linuxkit/bar:123abc`, `linuxkit/baz:aabb666`, and `linuxkit/qux:bbcc777`, this will create the following build args: + +``` +DEP_HASH_LINUXKIT_BAR=linuxkit/bar:123abc +DEP_HASH_LINUXKIT_BAZ=linuxkit/baz:aabb666 +DEP_HASH_LINUXKIT_QUX=linuxkit/qux:bbcc777 +``` + ## Releases Normally, whenever a package is updated, CI will build and push the package to Docker Hub by calling `linuxkit pkg push`. diff --git a/src/cmd/linuxkit/pkg_build.go b/src/cmd/linuxkit/pkg_build.go index 88055dbae..288d3f66c 100644 --- a/src/cmd/linuxkit/pkg_build.go +++ b/src/cmd/linuxkit/pkg_build.go @@ -129,7 +129,7 @@ func pkgBuildCmd() *cobra.Command { return fmt.Errorf("error transforming build arg %s: %v", line, err) } - buildArgs = append(buildArgs, buildArg) + buildArgs = append(buildArgs, buildArg...) } if err := scanner.Err(); err != nil { return fmt.Errorf("error reading build args file %s: %w", filename, err) @@ -138,9 +138,9 @@ func pkgBuildCmd() *cobra.Command { opts = append(opts, pkglib.WithBuildArgs(buildArgs)) // also need to parse the build args from the build.yml file for any special linuxkit values - for _, p := range pkgs { - if err := p.ProcessBuildArgs(); err != nil { - return fmt.Errorf("error processing build args for package %q: %w", p.Tag(), err) + for i := range pkgs { + if err := pkgs[i].ProcessBuildArgs(); err != nil { + return fmt.Errorf("error processing build args for package %q: %w", pkgs[i].Tag(), err) } } diff --git a/src/cmd/linuxkit/pkglib/buildarg.go b/src/cmd/linuxkit/pkglib/buildarg.go index 162fed6b0..e0c84026f 100644 --- a/src/cmd/linuxkit/pkglib/buildarg.go +++ b/src/cmd/linuxkit/pkglib/buildarg.go @@ -2,6 +2,7 @@ package pkglib import ( "fmt" + "os" "path/filepath" "strings" ) @@ -9,22 +10,24 @@ import ( const ( buildArgSpecialPrefix = "@lkt:" buildArgPkgPrefix = "pkg:" + buildArgsPkgPrefix = "pkgs:" + buildArgsKeyStemChar = "%" ) // TransformBuildArgValue transforms a build arg pair whose value starts with the special linuxkit prefix. -func TransformBuildArgValue(line, anchorFile string) (string, error) { +func TransformBuildArgValue(line, anchorFile string) ([]string, error) { parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { - return "", fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line) + return nil, fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line) } key := parts[0] val := parts[1] // check if the value is a special linuxkit value if !strings.HasPrefix(val, buildArgSpecialPrefix) { - return line, nil + return []string{line}, nil } stripped := strings.TrimPrefix(val, buildArgSpecialPrefix) - var final string + var final []string // see if we know what kind of value it is switch { case strings.HasPrefix(stripped, buildArgPkgPrefix): @@ -33,23 +36,74 @@ func TransformBuildArgValue(line, anchorFile string) (string, error) { if !strings.HasPrefix(pkgPath, "/") { anchorDir, err := filepath.Abs(filepath.Dir(anchorFile)) if err != nil { - return "", fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err) + return nil, fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err) } pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath)) } pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, pkgPath) if err != nil { - return "", err + return nil, err } if len(pkgs) == 0 { - return "", fmt.Errorf("no package found at path %q", pkgPath) + return nil, fmt.Errorf("no package found at path %q", pkgPath) } p := pkgs[0] tag := p.Tag() - final = tag + final = append(final, fmt.Sprintf("%s=%s", key, tag)) + case strings.HasPrefix(stripped, buildArgsPkgPrefix): + // validate the key + if !strings.Contains(key, buildArgsKeyStemChar) { + return nil, fmt.Errorf("invalid build arg key %q, must contain a '%s'", key, buildArgsKeyStemChar) + } + pkgPath := strings.TrimPrefix(stripped, buildArgsPkgPrefix) + if !strings.HasPrefix(pkgPath, "/") { + anchorDir, err := filepath.Abs(filepath.Dir(anchorFile)) + if err != nil { + return nil, fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err) + } + pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath)) + } + matches, err := filepath.Glob(pkgPath) + if err != nil { + return nil, fmt.Errorf("error globbing package path %q: %v", pkgPath, err) + } + if len(matches) == 0 { + return nil, fmt.Errorf("no packages found matching path %q", pkgPath) + } + var finalMatches []string + for _, match := range matches { + // ensure the match is a directory + info, err := os.Stat(match) + if err != nil { + return nil, fmt.Errorf("error stating package path %q: %v", match, err) + } + if !info.IsDir() { + continue + } + if strings.HasPrefix(info.Name(), ".") { + continue + } + finalMatches = append(finalMatches, match) + } + pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, finalMatches...) + if err != nil { + return nil, fmt.Errorf("error loading packages from paths %q: %v", pkgPath, err) + } + if len(pkgs) == 0 { + return nil, fmt.Errorf("no packages found at path %q", pkgPath) + } + for _, p := range pkgs { + tag := p.Tag() + // generate the special build arg key + image := strings.ReplaceAll(p.Image(), "/", "_") + image = strings.ReplaceAll(image, "-", "_") + image = strings.ToUpper(image) + updatedKey := strings.ReplaceAll(key, buildArgsKeyStemChar, image) + final = append(final, fmt.Sprintf("%s=%s", updatedKey, tag)) + } default: // something unknown - return "", fmt.Errorf("unknown linuxkit build arg value %q", val) + return nil, fmt.Errorf("unknown linuxkit build arg value %q", val) } - return fmt.Sprintf("%s=%s", key, final), nil + return final, nil } diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 4a7146d36..7983feb82 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -366,16 +366,21 @@ func (p Pkg) cleanForBuild() error { return nil } -func (p Pkg) ProcessBuildArgs() error { +func (p *Pkg) ProcessBuildArgs() error { if p.buildArgs == nil { return nil } - var err error - for i, arg := range *p.buildArgs { - (*p.buildArgs)[i], err = TransformBuildArgValue(arg, p.buildYML) + var buildArgs []string + for _, arg := range *p.buildArgs { + transformedLine, err := TransformBuildArgValue(arg, p.buildYML) if err != nil { return fmt.Errorf("error processing build arg %q: %v", arg, err) } + buildArgs = append(buildArgs, transformedLine...) + } + // Replace the original build args with the transformed ones + if len(buildArgs) > 0 { + p.buildArgs = &buildArgs } return nil } diff --git a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh index b11e61ec9..b5aaad9ff 100644 --- a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh +++ b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh @@ -19,10 +19,6 @@ clean_up() { } trap clean_up EXIT -# to be clear -pwd -ls -la . - for i in "${TMPDIR1}" "${TMPDIR2}"; do rm -rf "${i}" mkdir -p "${i}" @@ -57,7 +53,6 @@ current=$(linuxkit pkg show-tag .) # dump it to a filesystem linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf - # for extra debugging -find "${TMPEXPORT}" -type f -exec ls -la {} \; actual1=$(cat ${TMPEXPORT}/var/hash1) actual2=$(cat ${TMPEXPORT}/var/hash2) diff --git a/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh b/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh index 5e961ef23..19084a932 100644 --- a/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh +++ b/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh @@ -19,10 +19,6 @@ clean_up() { } trap clean_up EXIT -# to be clear -pwd -ls -la . - for i in "${TMPDIR1}" "${TMPDIR2}"; do rm -rf "${i}" mkdir -p "${i}" @@ -57,7 +53,6 @@ current=$(linuxkit pkg show-tag .) # dump it to a filesystem linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf - # for extra debugging -find "${TMPEXPORT}" -type f -exec ls -la {} \; actual1=$(cat ${TMPEXPORT}/var/hash1) actual2=$(cat ${TMPEXPORT}/var/hash2) diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/.gitignore b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/.gitignore new file mode 100644 index 000000000..ca8089a1f --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/.gitignore @@ -0,0 +1 @@ +foo/ \ No newline at end of file diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/Dockerfile b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/Dockerfile new file mode 100644 index 000000000..730c9b18e --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.21 +ARG HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_ONE +ARG HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_TWO +RUN printf '%s\n' "${HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_ONE}" > /var/hash1 +RUN printf '%s\n' "${HASH_LINUXKIT_HASHES_IN_BUILD_ARGS_TWO}" > /var/hash2 diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-args b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-args new file mode 100644 index 000000000..f7f9d6ea7 --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-args @@ -0,0 +1 @@ +HASH_%=@lkt:pkgs:/tmp/foo/* diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-file.yml b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-file.yml new file mode 100644 index 000000000..cdd4451a8 --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build-file.yml @@ -0,0 +1,2 @@ +org: linuxkit +image: hashes-in-build-args-file diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build.yml b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build.yml new file mode 100644 index 000000000..ea806d04c --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/build.yml @@ -0,0 +1,4 @@ +org: linuxkit +image: hashes-in-build-args-yaml +buildArgs: +- HASH_%=@lkt:pkgs:/tmp/foo/* diff --git a/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/test.sh b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/test.sh new file mode 100644 index 000000000..30cc9ebf0 --- /dev/null +++ b/test/cases/000_build/056_build_args/010_build_arg_special_multiple_yaml/test.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# SUMMARY: Check that tar output format build is reproducible +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +# need to build the special dir /tmp/bar12345 first +TMPDIR=/tmp/foo +TMPDIR1=${TMPDIR}/one +TMPDIR2=${TMPDIR}/two +TMPEXPORT=$(mktemp -d) +CACHE_DIR=$(mktemp -d) + +clean_up() { + rm -rf ${TMPDIR} ${CACHE_DIR} ${TMPEXPORT} +} +trap clean_up EXIT + +for i in "${TMPDIR1}" "${TMPDIR2}"; do + rm -rf "${i}" + mkdir -p "${i}" + echo "This is a test file for the special build arg" > "${i}/test" + cat > "${i}/build.yml" <&1 + if [ $? -ne 0 ]; then + echo "Build failed" + exit 1 + fi + + current=$(linuxkit pkg show-tag --build-yml "${yml}" .) + + # for debugging + find ${CACHE_DIR} -ls + cat ${CACHE_DIR}/index.json + index=$(cat ${CACHE_DIR}/index.json | jq -r '.manifests[] | select(.annotations["org.opencontainers.image.ref.name"] == "'${current}'") | .digest' | cut -d: -f2) + echo "Current package index: ${index}" + cat ${CACHE_DIR}/blobs/sha256/${index} | jq '.' + manifest=$(cat ${CACHE_DIR}/blobs/sha256/${index} | jq -r '.manifests[] | select(.platform.architecture == "'${targetarch}'") | .digest' | cut -d: -f2) + echo "Current package manifest: ${manifest}" + cat ${CACHE_DIR}/blobs/sha256/${manifest} | jq '.' + + + # dump it to a filesystem + rm -rf "${TMPEXPORT}/*" + linuxkit --cache ${CACHE_DIR} cache export --platform linux/${targetarch} --format filesystem --outfile /tmp/lktout123.tar "${current}" + file /tmp/lktout123.tar + ls -l /tmp/lktout123.tar + cat /tmp/lktout123.tar | tar -C "${TMPEXPORT}" -xvf - + # for extra debugging + actual1=$(cat ${TMPEXPORT}/var/hash1) + actual2=$(cat ${TMPEXPORT}/var/hash2) + + + if [ "${expected1}" != "${actual1}" ]; then + echo "Expected HASH1: ${expected1}, but got: ${actual1}" + exit 1 + fi + + if [ "${expected2}" != "${actual2}" ]; then + echo "Expected HASH2: ${expected2}, but got: ${actual2}" + exit 1 + fi +done + +exit 0