diff --git a/docs/packages.md b/docs/packages.md index 792dca7c3..3b223c770 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -57,7 +57,7 @@ A package source consists of a directory containing at least two files: - `gitrepo` _(string)_: The git repository where the package source is kept. - `network` _(bool)_: Allow network access during the package build (default: no) - `disable-cache` _(bool)_: Disable build cache for this package (default: no) -- `buildArgs` will forward a list of build arguments down to docker. As if `--build-arg` was specified during `docker build` +- `buildArgs` will forward a list of build arguments down to docker. As if `--build-arg` was specified during `docker build`. See [BuildArgs][BuildArgs] for more information. - `config`: _(struct `github.com/moby/tool/src/moby.ImageConfig`)_: Image configuration, marshalled to JSON and added as `org.mobyproject.config` label on image (default: no label) - `depends`: Contains information on prerequisites which must be satisfied in order to build the package. Has subfields: - `docker-images`: Docker images to be made available (as `tar` files via `docker image save`) within the package build context. Contains the following nested fields: @@ -382,6 +382,37 @@ ARG all_proxy LinuxKit does not judge between lower-cased or upper-cased variants of these options, e.g. `http_proxy` vs `HTTP_PROXY`, as `docker build` does not either. It just passes them through "as-is". +## Build Args + +`linuxkit` does not support passing random CLI flags for build arguments when building packages. +This is inline with its philosophy, of having as reproducible builds as possible, which requires +everything to be available on disk and in the repository. + +It is possible to bypass this, but this is not recommended. + +As described in [Preset build arguments][Preset build arguments], linuxkit automatically sets some build arguments +when building packages. However, you can also set your own build arguments, which will be passed to the +`docker build` command. +You can include your own build args in several ways. + +* `build.yml` - you can add a `buildArgs` field to the `build.yml` file, which will be passed as `--build-arg` to `docker build`. +* `linuxkit pkg build` - you can pass the `--build-arg-file ` flag, with one `=` pair per line, which will be passed as `--build-arg` to `docker build`. + +When parsing for build args, whether from `build.yml`'s `buildArgs` field or from the `--build-arg-file`, +linuxkit has support for certain calculated build args for the value of the arg. You can set these using the following syntax. + +All calculated build args are prefixed with `@lkt:`. + +* `@lkt:pkg:` - the linuxkit package hash of the path, as determined by `linuxkit pkg show-tag `. The `` can be absolute, or if provided as a relative path, it is relative to the working directory of the file. For example, if provided in the `buildArgs` section of `build.yml`, it is relative to the package directory; if provided in `--build-arg-file `, it is relative to the directory in which exists. + +For example: + +```yaml +buildArgs: + - DEP_HASH=@lkt:pkg:/usr/local/foo # will be replaced with the value of `linuxkit pkg show-tag /usr/local/foo` + - REL_HASH=@lkt:pkg:foo # will be replaced with the value of `linuxkit pkg show-tag foo` relative to this build.yml file +``` + ## Releases Normally, whenever a package is updated, CI will build and push the package to Docker Hub by calling `linuxkit pkg push`. diff --git a/src/cmd/linuxkit/const.go b/src/cmd/linuxkit/const.go index 4d97129a3..0bd81879f 100644 --- a/src/cmd/linuxkit/const.go +++ b/src/cmd/linuxkit/const.go @@ -1,6 +1,10 @@ package main -const ( - defaultPkgBuildYML = "build.yml" - defaultPkgCommit = "HEAD" +import ( + "github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib" +) + +const ( + defaultPkgBuildYML = pkglib.DefaultPkgBuildYML + defaultPkgCommit = pkglib.DefaultPkgCommit ) diff --git a/src/cmd/linuxkit/pkg_build.go b/src/cmd/linuxkit/pkg_build.go index dd4533918..88055dbae 100644 --- a/src/cmd/linuxkit/pkg_build.go +++ b/src/cmd/linuxkit/pkg_build.go @@ -122,7 +122,14 @@ func pkgBuildCmd() *cobra.Command { defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) for scanner.Scan() { - buildArgs = append(buildArgs, scanner.Text()) + line := strings.TrimSpace(scanner.Text()) + // check if the value is a special linuxkit value + buildArg, err := pkglib.TransformBuildArgValue(line, filename) + if err != nil { + return fmt.Errorf("error transforming build arg %s: %v", line, err) + } + + buildArgs = append(buildArgs, buildArg) } if err := scanner.Err(); err != nil { return fmt.Errorf("error reading build args file %s: %w", filename, err) @@ -130,6 +137,13 @@ func pkgBuildCmd() *cobra.Command { } opts = append(opts, pkglib.WithBuildArgs(buildArgs)) + // also need to parse the build args from the build.yml file for any special linuxkit values + for _, p := range pkgs { + if err := p.ProcessBuildArgs(); err != nil { + return fmt.Errorf("error processing build args for package %q: %w", p.Tag(), err) + } + } + // skipPlatformsMap contains platforms that should be skipped skipPlatformsMap := make(map[string]bool) if skipPlatforms != "" { diff --git a/src/cmd/linuxkit/pkglib/buildarg.go b/src/cmd/linuxkit/pkglib/buildarg.go new file mode 100644 index 000000000..162fed6b0 --- /dev/null +++ b/src/cmd/linuxkit/pkglib/buildarg.go @@ -0,0 +1,55 @@ +package pkglib + +import ( + "fmt" + "path/filepath" + "strings" +) + +const ( + buildArgSpecialPrefix = "@lkt:" + buildArgPkgPrefix = "pkg:" +) + +// TransformBuildArgValue transforms a build arg pair whose value starts with the special linuxkit prefix. +func TransformBuildArgValue(line, anchorFile string) (string, error) { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return "", fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", line) + } + key := parts[0] + val := parts[1] + // check if the value is a special linuxkit value + if !strings.HasPrefix(val, buildArgSpecialPrefix) { + return line, nil + } + stripped := strings.TrimPrefix(val, buildArgSpecialPrefix) + var final string + // see if we know what kind of value it is + switch { + case strings.HasPrefix(stripped, buildArgPkgPrefix): + pkgPath := strings.TrimPrefix(stripped, buildArgPkgPrefix) + // see if it is an absolute or relative path + if !strings.HasPrefix(pkgPath, "/") { + anchorDir, err := filepath.Abs(filepath.Dir(anchorFile)) + if err != nil { + return "", fmt.Errorf("error getting absolute path for anchor file %q: %v", anchorFile, err) + } + pkgPath = filepath.Clean(filepath.Join(anchorDir, pkgPath)) + } + pkgs, err := NewFromConfig(PkglibConfig{BuildYML: DefaultPkgBuildYML, HashCommit: DefaultPkgCommit}, pkgPath) + if err != nil { + return "", err + } + if len(pkgs) == 0 { + return "", fmt.Errorf("no package found at path %q", pkgPath) + } + p := pkgs[0] + tag := p.Tag() + final = tag + default: + // something unknown + return "", fmt.Errorf("unknown linuxkit build arg value %q", val) + } + return fmt.Sprintf("%s=%s", key, final), nil +} diff --git a/src/cmd/linuxkit/pkglib/const.go b/src/cmd/linuxkit/pkglib/const.go new file mode 100644 index 000000000..44e4264f2 --- /dev/null +++ b/src/cmd/linuxkit/pkglib/const.go @@ -0,0 +1,8 @@ +package pkglib + +const ( + // DefaultPkgBuildYML is the default name of the package build file + DefaultPkgBuildYML = "build.yml" + // DefaultPkgCommit is the default commit to use for packages + DefaultPkgCommit = "HEAD" +) diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index c34924d92..4a7146d36 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -92,6 +92,7 @@ type Pkg struct { // Internal state path string + buildYML string // full path to the build.yml file, not just relative to path dockerfile string hash string tag string @@ -150,7 +151,8 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) { return nil, err } - b, err := os.ReadFile(filepath.Join(pkgPath, cfg.BuildYML)) + buildYmlFile := filepath.Join(pkgPath, cfg.BuildYML) + b, err := os.ReadFile(buildYmlFile) if err != nil { return nil, err } @@ -292,6 +294,7 @@ func NewFromConfig(cfg PkglibConfig, args ...string) ([]Pkg, error) { dockerDepends: dockerDepends, dirty: dirty, path: pkgPath, + buildYML: buildYmlFile, dockerfile: pi.Dockerfile, git: git, tag: tag, @@ -363,6 +366,20 @@ func (p Pkg) cleanForBuild() error { return nil } +func (p Pkg) ProcessBuildArgs() error { + if p.buildArgs == nil { + return nil + } + var err error + for i, arg := range *p.buildArgs { + (*p.buildArgs)[i], err = TransformBuildArgValue(arg, p.buildYML) + if err != nil { + return fmt.Errorf("error processing build arg %q: %v", arg, err) + } + } + return nil +} + // Expands path from relative to abs against base, ensuring the result is within base, but is not base itself. Field is the fieldname, to be used for constructing the error. func makeAbsSubpath(field, base, path string) (string, error) { if path == "" { diff --git a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/.gitignore b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/.gitignore new file mode 100644 index 000000000..ca8089a1f --- /dev/null +++ b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/.gitignore @@ -0,0 +1 @@ +foo/ \ No newline at end of file diff --git a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/Dockerfile b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/Dockerfile new file mode 100644 index 000000000..37c4229b1 --- /dev/null +++ b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.21 +ARG HASH1 +ARG HASH2 +RUN printf '%s\n' "${HASH1}" > /var/hash1 +RUN printf '%s\n' "${HASH2}" > /var/hash2 diff --git a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/build.yml b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/build.yml new file mode 100644 index 000000000..af98219d1 --- /dev/null +++ b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/build.yml @@ -0,0 +1,5 @@ +org: linuxkit +image: hashes-in-build-args +buildArgs: +- HASH1=@lkt:pkg:foo/ +- HASH2=@lkt:pkg:/tmp/bar12345/ diff --git a/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh new file mode 100644 index 000000000..b11e61ec9 --- /dev/null +++ b/test/cases/000_build/056_build_args/005_build_arg_special_yaml/test.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# SUMMARY: Check that tar output format build is reproducible +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +# need to build the special dir /tmp/bar12345 first +TMPDIR1=/tmp/bar12345 +TMPDIR2=./foo/ +TMPEXPORT=$(mktemp -d) +CACHE_DIR=$(mktemp -d) + +clean_up() { + rm -rf ${TMPDIR1} ${TMPDIR2} ${CACHE_DIR} ${TMPEXPORT} +} +trap clean_up EXIT + +# to be clear +pwd +ls -la . + +for i in "${TMPDIR1}" "${TMPDIR2}"; do + rm -rf "${i}" + mkdir -p "${i}" + echo "This is a test file for the special build arg" > "${i}/test" + cat > "${i}/build.yml" <&1) +if [ $? -ne 0 ]; then + echo "Build failed with logs:" + echo "${logs}" + exit 1 +fi + +expected1=$(linuxkit pkg show-tag "${TMPDIR1}") +expected2=$(linuxkit pkg show-tag "${TMPDIR2}") +current=$(linuxkit pkg show-tag .) + +# dump it to a filesystem +linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf - +# for extra debugging +find "${TMPEXPORT}" -type f -exec ls -la {} \; +actual1=$(cat ${TMPEXPORT}/var/hash1) +actual2=$(cat ${TMPEXPORT}/var/hash2) + + +if [ "${expected1}" != "${actual1}" ]; then + echo "Expected HASH1: ${expected1}, but got: ${actual1}" + exit 1 +fi + +if [ "${expected2}" != "${actual2}" ]; then + echo "Expected HASH2: ${expected2}, but got: ${actual2}" + exit 1 +fi + +# Check that the build args were correctly transformed + +exit 0 diff --git a/test/cases/000_build/056_build_args/006_build_arg_special_cli/Dockerfile b/test/cases/000_build/056_build_args/006_build_arg_special_cli/Dockerfile new file mode 100644 index 000000000..37c4229b1 --- /dev/null +++ b/test/cases/000_build/056_build_args/006_build_arg_special_cli/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.21 +ARG HASH1 +ARG HASH2 +RUN printf '%s\n' "${HASH1}" > /var/hash1 +RUN printf '%s\n' "${HASH2}" > /var/hash2 diff --git a/test/cases/000_build/056_build_args/006_build_arg_special_cli/build-args b/test/cases/000_build/056_build_args/006_build_arg_special_cli/build-args new file mode 100644 index 000000000..3956bfb0e --- /dev/null +++ b/test/cases/000_build/056_build_args/006_build_arg_special_cli/build-args @@ -0,0 +1,2 @@ +HASH1=@lkt:pkg:foo/ +HASH2=@lkt:pkg:/tmp/bar67890/ diff --git a/test/cases/000_build/056_build_args/006_build_arg_special_cli/build.yml b/test/cases/000_build/056_build_args/006_build_arg_special_cli/build.yml new file mode 100644 index 000000000..87c0bb62f --- /dev/null +++ b/test/cases/000_build/056_build_args/006_build_arg_special_cli/build.yml @@ -0,0 +1,2 @@ +org: linuxkit +image: test-image-in-yaml-build-args diff --git a/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh b/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh new file mode 100644 index 000000000..5e961ef23 --- /dev/null +++ b/test/cases/000_build/056_build_args/006_build_arg_special_cli/test.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# SUMMARY: Check that tar output format build is reproducible +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +# need to build the special dir /tmp/bar12345 first +TMPDIR1=/tmp/bar67890 +TMPDIR2=./foo/ +TMPEXPORT=$(mktemp -d) +CACHE_DIR=$(mktemp -d) + +clean_up() { + rm -rf ${TMPDIR1} ${TMPDIR2} ${CACHE_DIR} ${TMPEXPORT} +} +trap clean_up EXIT + +# to be clear +pwd +ls -la . + +for i in "${TMPDIR1}" "${TMPDIR2}"; do + rm -rf "${i}" + mkdir -p "${i}" + echo "This is a test file for the special build arg" > "${i}/test" + cat > "${i}/build.yml" <&1) +if [ $? -ne 0 ]; then + echo "Build failed with logs:" + echo "${logs}" + exit 1 +fi + +expected1=$(linuxkit pkg show-tag "${TMPDIR1}") +expected2=$(linuxkit pkg show-tag "${TMPDIR2}") +current=$(linuxkit pkg show-tag .) + +# dump it to a filesystem +linuxkit --cache ${CACHE_DIR} cache export --format filesystem --outfile - "${current}" | tar -C "${TMPEXPORT}" -xvf - +# for extra debugging +find "${TMPEXPORT}" -type f -exec ls -la {} \; +actual1=$(cat ${TMPEXPORT}/var/hash1) +actual2=$(cat ${TMPEXPORT}/var/hash2) + +if [ "${expected1}" != "${actual1}" ]; then + echo "Expected HASH1: ${expected1}, but got: ${actual1}" + exit 1 +fi + +if [ "${expected2}" != "${actual2}" ]; then + echo "Expected HASH2: ${expected2}, but got: ${actual2}" + exit 1 +fi + +# Check that the build args were correctly transformed + +exit 0