when comparing builder config files, ensure only compare post-processed

Signed-off-by: Avi Deitcher <avi@deitcher.net>

src/cmd/linuxkit/pkglib/dockerimpl.go

@@ -57,13 +57,14 @@ import (
 )
 
 const (
-	buildkitBuilderName   = "linuxkit-builder"
-	buildkitSocketPath    = "/run/buildkit/buildkitd.sock"
-	buildkitWaitServer    = 30 // seconds
-	buildkitCheckInterval = 1  // seconds
-	sbomFrontEndKey       = "attest:sbom"
-	buildkitConfigDir     = "/etc/buildkit"
-	buildkitConfigPath    = buildkitConfigDir + "/buildkitd.toml"
+	buildkitBuilderName    = "linuxkit-builder"
+	buildkitSocketPath     = "/run/buildkit/buildkitd.sock"
+	buildkitWaitServer     = 30 // seconds
+	buildkitCheckInterval  = 1  // seconds
+	sbomFrontEndKey        = "attest:sbom"
+	buildkitConfigDir      = "/etc/buildkit"
+	buildkitConfigFileName = "buildkitd.toml"
+	buildkitConfigPath     = buildkitConfigDir + "/" + buildkitConfigFileName
 )
 
 type dockerRunnerImpl struct {
@@ -277,6 +278,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 	for range buildKitCheckRetryCount {
 		var b bytes.Buffer
 		var cid string
+		var filesToLoadIntoContainer map[string][]byte
 		if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
 			// we already have a container named "linuxkit-builder" in the provided context.
 			// get its state and config
@@ -295,16 +297,25 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 				// if it is provided, we assume it is false until proven true
 				log.Debugf("checking if configPath %s is correct in container %s", configPath, name)
 				configPathCorrect = false
-				if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "exec", name, "cat", buildkitConfigPath); err == nil {
+				var configB bytes.Buffer
+				// we cannot exactly use the local config file, as it gets modified to get loaded into the container
+				// so we preprocess it using the same library that would load it up
+				filesToLoadIntoContainer, err = confutil.LoadConfigFiles(configPath)
+				if err != nil {
+					return nil, fmt.Errorf("failed to load buildkit config file %s: %v", configPath, err)
+				}
+				if err := dr.command(nil, &configB, io.Discard, "--context", dockerContext, "container", "exec", name, "cat", buildkitConfigPath); err == nil {
 					// sha256sum the config file to see if it matches the provided configPath
-					containerConfigFileHash := sha256.Sum256(b.Bytes())
+					containerConfigFileHash := sha256.Sum256(configB.Bytes())
 					log.Debugf("container %s has configPath %s with sha256sum %x", name, buildkitConfigPath, containerConfigFileHash)
-					configFileContents, err := os.ReadFile(configPath)
-					if err != nil {
-						return nil, fmt.Errorf("unable to read buildkit config file %s: %v", configPath, err)
+					log.Tracef("container %s has configPath %s with contents:\n%s", name, buildkitConfigPath, configB.String())
+					configFileContents, ok := filesToLoadIntoContainer[buildkitConfigFileName]
+					if !ok {
+						return nil, fmt.Errorf("unable to read provided buildkit config file %s: %v", configPath, err)
 					}
 					localConfigFileHash := sha256.Sum256(configFileContents)
 					log.Debugf("local %s has configPath %s with sha256sum %x", name, configPath, localConfigFileHash)
+					log.Tracef("local %s has configPath %s with contents:\n%s", name, buildkitConfigPath, string(configFileContents))
 					if bytes.Equal(containerConfigFileHash[:], localConfigFileHash[:]) {
 						log.Debugf("configPath %s in container %s matches local configPath %s", buildkitConfigPath, name, configPath)
 						configPathCorrect = true
@@ -314,8 +325,6 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 				} else {
 					log.Debugf("could not read configPath %s from container %s, assuming it is not correct", buildkitConfigPath, name)
 				}
-				// now rewrite and copy over certs, if needed
-				//https://github.com/docker/buildx/blob/master/util/confutil/container.go#L27
 			}
 
 			switch {
@@ -338,7 +347,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 				stop = isRunning
 				remove = true
 			case !configPathCorrect:
-				fmt.Printf("existing container has wrong configPath mount, restarting\n")
+				fmt.Printf("existing container has wrong configPath contents, restarting\n")
 				recreate = true
 				stop = isRunning
 				remove = true
@@ -405,11 +414,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 			}
 			// copy in the buildkit config file, if provided
 			if configPath != "" {
-				files, err := confutil.LoadConfigFiles(configPath)
-				if err != nil {
-					return nil, fmt.Errorf("failed to load buildkit config file %s: %v", configPath, err)
-				}
-				if err := dr.copyFilesToContainer(name, files); err != nil {
+				if err := dr.copyFilesToContainer(name, filesToLoadIntoContainer); err != nil {
 					return nil, fmt.Errorf("failed to copy buildkit config file %s and certificates into container %s: %v", configPath, name, err)
 				}
 			}

test/cases/040_packages/026_buildkit_config/.gitignore

@@ -0,0 +1 @@
+docker-config/

test/cases/040_packages/026_buildkit_config/Dockerfile

@@ -0,0 +1,2 @@
+FROM alpine:3.21
+RUN echo hi

test/cases/040_packages/026_buildkit_config/build.yml

@@ -0,0 +1,2 @@
+org: linuxkit
+image: builder-config

test/cases/040_packages/026_buildkit_config/buildkitd-2.toml

@@ -0,0 +1,18 @@
+# it does not matter what these contents are, as long as they are valid and can be processed
+# and are different than the ones in buildkitd.toml
+debug = true
+# trace = true
+insecure-entitlements = [ "network.host", "security.insecure" ]
+
+[worker.oci]
+  max-parallelism = 48
+[worker.oci.gcpulimits]
+  enabled = true
+
+[log]
+  # log formatter: json or text
+  format = "json"
+
+[registry."172.17.0.2:5001"]
+  insecure = true
+  http = true

test/cases/040_packages/026_buildkit_config/buildkitd.toml

@@ -0,0 +1,17 @@
+# it does not matter what these contents are, as long as they are valid and can be processed
+debug = true
+# trace = true
+insecure-entitlements = [ "network.host", "security.insecure" ]
+
+[worker.oci]
+  max-parallelism = 56
+[worker.oci.gcpulimits]
+  enabled = false
+
+[log]
+  # log formatter: json or text
+  format = "text"
+
+[registry."172.17.0.2:5000"]
+  insecure = true
+  http = true

test/cases/040_packages/026_buildkit_config/test.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# SUMMARY: Check that we can access a registry with auth
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+clean_up() {
+	[ -n "${CACHEDIR}" ] && rm -rf "${CACHEDIR}"
+}
+trap clean_up EXIT
+
+# determine platform
+ARCH=$(uname -m)
+if [ "${ARCH}" = "x86_64" ]; then
+  ARCH="amd64"
+elif [ "${ARCH}" = "aarch64" ]; then
+  ARCH="arm64"
+fi
+PLATFORM="linux/${ARCH}"
+
+CACHEDIR=$(mktemp -d)
+
+# tests:
+# 1. build the local package with the custom buildkitd.toml - should succeed
+# 2. rebuild the local package with the same buildkitd.toml - should succeed without starting a new builder container
+# 3. rebuild the local package with the different buildkitd-2.toml - should succeed after starting a new builder container
+if ! linuxkit --verbose 3 --cache "${CACHEDIR}" pkg build --platforms "${PLATFORM}" \
+  --builder-config "$(pwd)/buildkitd.toml" --force \
+  .; then 
+  echo "Build 1 failed"
+  exit 1
+fi
+CID1=$(docker inspect linuxkit-builder --format '{{.ID}}')
+
+# get the containerd
+
+if ! linuxkit --verbose 3 --cache "${CACHEDIR}" pkg build --platforms "${PLATFORM}" \
+  --builder-config "$(pwd)/buildkitd.toml" --force \
+  .; then 
+  echo "Build 2 failed"
+  exit 1
+fi
+CID2=$(docker inspect linuxkit-builder --format '{{.ID}}')
+
+if ! linuxkit --verbose 3 --cache "${CACHEDIR}" pkg build --platforms "${PLATFORM}" \
+  --builder-config "$(pwd)/buildkitd-2.toml" --force \
+  .; then 
+  echo "Build 3 failed"
+  exit 1
+fi
+CID3=$(docker inspect linuxkit-builder --format '{{.ID}}')
+
+# CID1 and CID2 should match, CID3 should not
+echo "CID1: ${CID1}"
+echo "CID2: ${CID2}"
+echo "CID3: ${CID3}"
+
+if [ "${CID1}" = "${CID2}" ] && [ "${CID2}" != "${CID3}" ]; then
+  echo "Build 1 and 2 used the same builder container, but Build 3 used a different one"
+else
+  echo "Unexpected builder container behavior"
+  exit 1
+fi
+
+exit 0