Merge pull request #4204 from rucoder/rucoder/per-user-builder-name

pkg build: make buildkit builder container name configurable
This commit is contained in:
Justin Cormack
2026-03-12 10:27:40 +00:00
committed by GitHub
8 changed files with 125 additions and 85 deletions

View File

@@ -16,6 +16,7 @@ import (
const (
buildersEnvVar = "LINUXKIT_BUILDERS"
envVarCacheDir = "LINUXKIT_CACHE"
envVarBuilderName = "LINUXKIT_BUILDER_NAME"
defaultBuilderImage = "moby/buildkit:v0.26.3"
)
@@ -40,6 +41,7 @@ func pkgBuildCmd() *cobra.Command {
platforms string
skipPlatforms string
builders string
builderName = flagOverEnvVarOverDefaultString{def: pkglib.DefaultBuilderName(), envVar: envVarBuilderName}
builderImage string
builderConfig string
builderRestart bool
@@ -195,12 +197,15 @@ func pkgBuildCmd() *cobra.Command {
if _, err := os.Stat(builderConfig); err != nil {
return fmt.Errorf("error reading builder config file %s: %w", builderConfig, err)
}
opts = append(opts, pkglib.WithBuildBuilderConfig(builderConfig))
}
opts = append(opts, pkglib.WithBuildBuilders(buildersMap))
opts = append(opts, pkglib.WithBuildBuilderImage(builderImage))
opts = append(opts, pkglib.WithBuildBuilderRestart(builderRestart))
opts = append(opts, pkglib.WithBuildBuilderConfig(pkglib.BuilderConfig{
Name: builderName.String(),
Image: builderImage,
ConfigPath: builderConfig,
Restart: builderRestart,
}))
opts = append(opts, pkglib.WithProgress(progress))
if len(ssh) > 0 {
opts = append(opts, pkglib.WithSSH(ssh))
@@ -303,6 +308,7 @@ func pkgBuildCmd() *cobra.Command {
cmd.Flags().StringVar(&platforms, "platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
cmd.Flags().StringVar(&skipPlatforms, "skip-platforms", "", "Platforms that should be skipped, even if present in build.yml")
cmd.Flags().StringVar(&builders, "builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
cmd.Flags().Var(&builderName, "builder-name", fmt.Sprintf("Name of the buildkit builder container, default: %s, overrides env var %s", pkglib.DefaultBuilderName(), envVarBuilderName))
cmd.Flags().StringVar(&builderImage, "builder-image", defaultBuilderImage, "buildkit builder container image to use")
cmd.Flags().StringVar(&builderConfig, "builder-config", "", "path to buildkit builder config.toml file to use, overrides the default config.toml in the builder image. When provided, copied over into builder, along with all certs. Use paths for certificates relative to your local host, they will be adjusted on copying into the container. USE WITH CAUTION")
cmd.Flags().BoolVar(&builderRestart, "builder-restart", false, "force restarting builder, even if container with correct name and image exists")

View File

@@ -13,6 +13,7 @@ import (
func pkgBuilderCmd() *cobra.Command {
var (
builders string
builderName = flagOverEnvVarOverDefaultString{def: pkglib.DefaultBuilderName(), envVar: envVarBuilderName}
platforms string
builderImage string
builderConfigPath string
@@ -39,13 +40,18 @@ func pkgBuilderCmd() *cobra.Command {
}
platformsToClean := strings.Split(platforms, ",")
bc := pkglib.BuilderConfig{
Name: builderName.String(),
Image: builderImage,
ConfigPath: builderConfigPath,
}
switch command {
case "du":
if err := pkglib.DiskUsage(buildersMap, builderImage, builderConfigPath, platformsToClean, verbose); err != nil {
if err := pkglib.DiskUsage(buildersMap, bc, platformsToClean, verbose); err != nil {
return fmt.Errorf("unable to print disk usage of builder: %w", err)
}
case "prune":
if err := pkglib.PruneBuilder(buildersMap, builderImage, builderConfigPath, platformsToClean, verbose); err != nil {
if err := pkglib.PruneBuilder(buildersMap, bc, platformsToClean, verbose); err != nil {
return fmt.Errorf("unable to prune builder: %w", err)
}
default:
@@ -56,6 +62,7 @@ func pkgBuilderCmd() *cobra.Command {
}
cmd.PersistentFlags().StringVar(&builders, "builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes")
cmd.PersistentFlags().Var(&builderName, "builder-name", fmt.Sprintf("Name of the buildkit builder container, default: %s, overrides env var %s", pkglib.DefaultBuilderName(), envVarBuilderName))
cmd.PersistentFlags().StringVar(&platforms, "platforms", fmt.Sprintf("linux/%s", runtime.GOARCH), "Which platforms we built images for")
cmd.PersistentFlags().StringVar(&builderImage, "builder-image", defaultBuilderImage, "buildkit builder container image to use")
cmd.Flags().StringVar(&builderConfigPath, "builder-config", "", "path to buildkit builder config.toml file to use, overrides the default config.toml in the builder image; USE WITH CAUTION")

View File

@@ -24,32 +24,30 @@ import (
)
type buildOpts struct {
skipBuild bool
force bool
pull bool
ignoreCache bool
push bool
dryRun bool
preCacheImages bool
release string
manifest bool
targetDocker bool
cacheDir string
cacheProvider spec.CacheProvider
platforms []imagespec.Platform
builders map[string]string
runner DockerRunner
writer io.Writer
builderImage string
builderConfigPath string
builderRestart bool
sbomScan bool
sbomScannerImage string
dockerfile string
buildArgs []string
progress string
ssh []string
registryAuth map[string]spec.RegistryAuth
skipBuild bool
force bool
pull bool
ignoreCache bool
push bool
dryRun bool
preCacheImages bool
release string
manifest bool
targetDocker bool
cacheDir string
cacheProvider spec.CacheProvider
platforms []imagespec.Platform
builders map[string]string
runner DockerRunner
writer io.Writer
builderConfig BuilderConfig
sbomScan bool
sbomScannerImage string
dockerfile string
buildArgs []string
progress string
ssh []string
registryAuth map[string]spec.RegistryAuth
}
// BuildOpt allows callers to specify options to Build
@@ -160,26 +158,10 @@ func WithBuildOutputWriter(w io.Writer) BuildOpt {
}
}
// WithBuildBuilderImage set the builder container image to use.
func WithBuildBuilderImage(image string) BuildOpt {
// WithBuildBuilderConfig set the builder configuration to use.
func WithBuildBuilderConfig(bc BuilderConfig) BuildOpt {
return func(bo *buildOpts) error {
bo.builderImage = image
return nil
}
}
// WithBuildBuilderConfig set the contents of the
func WithBuildBuilderConfig(builderConfigPath string) BuildOpt {
return func(bo *buildOpts) error {
bo.builderConfigPath = builderConfigPath
return nil
}
}
// WithBuildBuilderRestart restart the builder container even if it already is running with the correct image version
func WithBuildBuilderRestart(restart bool) BuildOpt {
return func(bo *buildOpts) error {
bo.builderRestart = restart
bo.builderConfig = bc
return nil
}
}
@@ -322,7 +304,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
case d == nil && bo.dryRun:
d = NewDockerDryRunner()
case d == nil:
d = NewDockerRunner(p.cache)
d = NewDockerRunner(p.cache, bo.builderConfig)
}
c := bo.cacheProvider
@@ -492,7 +474,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
// build for each arch and save in the linuxkit cache
for _, platform := range platformsToBuild {
builtDescs, err := p.buildArch(ctx, d, c, bo.builderImage, bo.builderConfigPath, platform.Architecture, bo.builderRestart, writer, bo, imageBuildOpts)
builtDescs, err := p.buildArch(ctx, d, c, platform.Architecture, writer, bo, imageBuildOpts)
if err != nil {
return fmt.Errorf("error building for arch %s: %v", platform.Architecture, err)
}
@@ -645,7 +627,7 @@ func (p Pkg) Build(bos ...BuildOpt) error {
// C - manifest, saved in cache as is, referenced by the index (E), and returned as a descriptor
// D - attestations (if any), saved in cache as is, referenced by the index (E), and returned as a descriptor
// E - index, saved in cache as is, stored in cache as tag "image:tag-arch", *not* returned as a descriptor
func (p Pkg) buildArch(ctx context.Context, d DockerRunner, c spec.CacheProvider, builderImage, builderConfigPath, arch string, restart bool, writer io.Writer, bo buildOpts, imageBuildOpts spec.ImageBuildOptions) ([]registry.Descriptor, error) {
func (p Pkg) buildArch(ctx context.Context, d DockerRunner, c spec.CacheProvider, arch string, writer io.Writer, bo buildOpts, imageBuildOpts spec.ImageBuildOptions) ([]registry.Descriptor, error) {
var (
tagArch string
tag = p.FullTag()
@@ -674,8 +656,8 @@ func (p Pkg) buildArch(ctx context.Context, d DockerRunner, c spec.CacheProvider
return nil, err
}
// find the desired builder
builderName := getBuilderForPlatform(arch, bo.builders)
// find the desired docker context for the builder
dockerContext := getBuilderForPlatform(arch, bo.builders)
// set the target
var (
@@ -714,7 +696,7 @@ func (p Pkg) buildArch(ctx context.Context, d DockerRunner, c spec.CacheProvider
imageBuildOpts.Dockerfile = bo.dockerfile
if err := d.Build(ctx, tagArch, p.path, builderName, builderImage, builderConfigPath, platform, restart, bo.preCacheImages, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
if err := d.Build(ctx, tagArch, p.path, dockerContext, platform, bo.preCacheImages, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
stdoutCloser()
if strings.Contains(err.Error(), "executor failed running [/dev/.buildkit_qemu_emulator") {
return nil, fmt.Errorf("buildkit was unable to emulate %s. check binfmt has been set up and works for this platform: %v", platform, err)

View File

@@ -53,10 +53,10 @@ func (d *dockerMocker) ContextSupportCheck() error {
}
return errors.New("contexts not supported")
}
func (d *dockerMocker) Builder(_ context.Context, _, _, _, _ string, _ bool) (*buildkitClient.Client, error) {
func (d *dockerMocker) Builder(_ context.Context, _, _ string) (*buildkitClient.Client, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *dockerMocker) Build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, builderRestart, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
func (d *dockerMocker) Build(ctx context.Context, tag, pkg, dockerContext, platform string, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
if !d.enableBuild {
return errors.New("build disabled")
}

View File

@@ -18,12 +18,25 @@ import (
_ "github.com/moby/buildkit/client/connhelper/ssh"
)
// BuilderConfig holds configuration for the buildkit builder container.
type BuilderConfig struct {
// Name is the container name for the buildkit builder (e.g., "linuxkit-builder-alice").
Name string
// Image is the container image to run (e.g., "moby/buildkit:v0.26.3").
Image string
// ConfigPath is an optional path to a buildkitd.toml config file.
ConfigPath string
// Restart forces recreation of the builder container even if one with the
// correct name and image already exists.
Restart bool
}
type DockerRunner interface {
Tag(ref, tag string) error
Build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
Build(ctx context.Context, tag, pkg, dockerContext, platform string, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
Save(tgt string, refs ...string) error
Load(src io.Reader) error
Pull(img string) (bool, error)
ContextSupportCheck() error
Builder(ctx context.Context, dockerContext, builderImage, builderConfigPath, platform string, restart bool) (*buildkitClient.Client, error)
Builder(ctx context.Context, dockerContext, platform string) (*buildkitClient.Client, error)
}

View File

@@ -46,7 +46,7 @@ func (dr *dockerDryRunnerImpl) ContextSupportCheck() error {
// 1. if dockerContext is provided, try to create a builder with that context; if it succeeds, we are done; if not, return an error.
// 2. try to find an existing named runner with the pattern; if it succeeds, we are done; if not, try next.
// 3. try to create a generic builder using the default context named "linuxkit".
func (dr *dockerDryRunnerImpl) Builder(ctx context.Context, dockerContext, builderImage, builderConfigPath, platform string, restart bool) (*buildkitClient.Client, error) {
func (dr *dockerDryRunnerImpl) Builder(ctx context.Context, dockerContext, platform string) (*buildkitClient.Client, error) {
return nil, nil
}
@@ -58,7 +58,7 @@ func (dr *dockerDryRunnerImpl) Tag(ref, tag string) error {
return errors.New("not implemented")
}
func (dr *dockerDryRunnerImpl) Build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart, preCacheImages bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
func (dr *dockerDryRunnerImpl) Build(ctx context.Context, tag, pkg, dockerContext, platform string, preCacheImages bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
// build args
var buildArgs []string
for k, v := range imageBuildOpts.BuildArgs {

View File

@@ -67,12 +67,27 @@ const (
buildkitConfigPath = buildkitConfigDir + "/" + buildkitConfigFileName
)
type dockerRunnerImpl struct {
cache bool
// DefaultBuilderName returns the default builder container name.
func DefaultBuilderName() string {
return buildkitBuilderName
}
func NewDockerRunner(cache bool) DockerRunner {
return &dockerRunnerImpl{cache: cache}
// builderVolumeName returns the named volume used to persist buildkit state
// (build cache, snapshots, content store) across container recreations.
func builderVolumeName(containerName string) string {
return containerName + "-state"
}
type dockerRunnerImpl struct {
cache bool
builder BuilderConfig
}
func NewDockerRunner(cache bool, bc BuilderConfig) DockerRunner {
if bc.Name == "" {
bc.Name = DefaultBuilderName()
}
return &dockerRunnerImpl{cache: cache, builder: bc}
}
func isExecErrNotFound(err error) bool {
@@ -219,14 +234,15 @@ func (dr *dockerRunnerImpl) ContextSupportCheck() error {
// 1. if dockerContext is provided, try to create a builder with that context; if it succeeds, we are done; if not, return an error.
// 2. try to find an existing named runner with the pattern; if it succeeds, we are done; if not, try next.
// 3. try to create a generic builder using the default context named "linuxkit".
func (dr *dockerRunnerImpl) Builder(ctx context.Context, dockerContext, builderImage, builderConfigPath, platform string, restart bool) (*buildkitClient.Client, error) {
func (dr *dockerRunnerImpl) Builder(ctx context.Context, dockerContext, platform string) (*buildkitClient.Client, error) {
bc := dr.builder
// if we were given a context, we must find a builder and use it, or create one and use it
if dockerContext != "" {
// does the context exist?
if err := dr.command(nil, io.Discard, io.Discard, "context", "inspect", dockerContext); err != nil {
return nil, fmt.Errorf("provided docker context '%s' not found", dockerContext)
}
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, platform, dockerContext, restart)
client, err := dr.builderEnsureContainer(ctx, bc.Name, bc.Image, bc.ConfigPath, platform, dockerContext, bc.Restart)
if err != nil {
return nil, fmt.Errorf("error preparing builder based on context '%s': %v", dockerContext, err)
}
@@ -237,13 +253,13 @@ func (dr *dockerRunnerImpl) Builder(ctx context.Context, dockerContext, builderI
dockerContext = fmt.Sprintf("%s-%s", "linuxkit", strings.ReplaceAll(platform, "/", "-"))
if err := dr.command(nil, io.Discard, io.Discard, "context", "inspect", dockerContext); err == nil {
// we found an appropriately named context, so let us try to use it or error out
if client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, platform, dockerContext, restart); err == nil {
if client, err := dr.builderEnsureContainer(ctx, bc.Name, bc.Image, bc.ConfigPath, platform, dockerContext, bc.Restart); err == nil {
return client, nil
}
}
// create a generic builder
client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, builderConfigPath, "", "default", restart)
client, err := dr.builderEnsureContainer(ctx, bc.Name, bc.Image, bc.ConfigPath, "", "default", bc.Restart)
if err != nil {
return nil, fmt.Errorf("error ensuring builder container in default context: %v", err)
}
@@ -262,9 +278,10 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
// recreate by default (true) unless we already have one that meets all of the requirements - image, permissions, etc.
recreate = true
// stop existing one
stop = false
remove = false
found = false
stop = false
remove = false
removeVolume = false
found = false
)
const (
@@ -335,11 +352,13 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
stop = isRunning
remove = true
case existingImage != image:
// if image mismatches, recreate
// if image mismatches, recreate and remove the state volume since we
// cannot guarantee buildkit state compatibility across versions
fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image)
recreate = true
stop = isRunning
remove = true
removeVolume = true
case !containerJSON[0].HostConfig.Privileged:
// if unprivileged, we need to remove it and start a new container with the right permissions
fmt.Printf("existing container %s is unprivileged, replacing\n", name)
@@ -394,11 +413,24 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
continue
}
}
if removeVolume {
volName := builderVolumeName(name)
fmt.Printf("removing builder state volume %s\n", volName)
// best-effort: volume may not exist yet on first run
_ = dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "volume", "rm", volName)
}
if recreate {
// create the builder
// this could be a single line, but it would be long. And it is easier to read when the
// docker command args, the image name, and the image args are all on separate lines.
args := []string{"--context", dockerContext, "container", "create", "--name", name, "--privileged"}
volName := builderVolumeName(name)
if removeVolume {
fmt.Printf("creating fresh builder state volume %s\n", volName)
} else {
fmt.Printf("reusing builder state volume %s\n", volName)
}
volMount := volName + ":/var/lib/buildkit"
args := []string{"--context", dockerContext, "container", "create", "--name", name, "--privileged", "-v", volMount}
args = append(args, image)
args = append(args, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug")
if configPath != "" {
@@ -507,9 +539,9 @@ func (dr *dockerRunnerImpl) Tag(ref, tag string) error {
return dr.command(nil, nil, nil, "image", "tag", ref, tag)
}
func (dr *dockerRunnerImpl) Build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart, preCacheImages bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
func (dr *dockerRunnerImpl) Build(ctx context.Context, tag, pkg, dockerContext, platform string, preCacheImages bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
// ensure we have a builder
client, err := dr.Builder(ctx, dockerContext, builderImage, builderConfigPath, platform, restart)
client, err := dr.Builder(ctx, dockerContext, platform)
if err != nil {
return fmt.Errorf("unable to ensure builder container: %v", err)
}

View File

@@ -93,14 +93,14 @@ func printVerbose(tw *tabwriter.Writer, du []*buildkitClient.UsageInfo) {
_ = tw.Flush()
}
func getClientForPlatform(ctx context.Context, buildersMap map[string]string, builderImage, builderConfigPath, platform string) (*buildkitClient.Client, error) {
func getClientForPlatform(ctx context.Context, buildersMap map[string]string, bc BuilderConfig, platform string) (*buildkitClient.Client, error) {
p, err := platforms.Parse(platform)
if err != nil {
return nil, fmt.Errorf("failed to parse platform: %s", err)
}
dr := NewDockerRunner(false)
builderName := getBuilderForPlatform(p.Architecture, buildersMap)
client, err := dr.Builder(ctx, builderName, builderImage, builderConfigPath, platform, false)
dr := NewDockerRunner(false, bc)
dockerContext := getBuilderForPlatform(p.Architecture, buildersMap)
client, err := dr.Builder(ctx, dockerContext, platform)
if err != nil {
return nil, fmt.Errorf("unable to ensure builder container: %v", err)
}
@@ -108,11 +108,11 @@ func getClientForPlatform(ctx context.Context, buildersMap map[string]string, bu
}
// DiskUsage of builder
func DiskUsage(buildersMap map[string]string, builderImage, builderConfigPath string, platformsToClean []string, verbose bool) error {
func DiskUsage(buildersMap map[string]string, bc BuilderConfig, platformsToClean []string, verbose bool) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for _, platform := range platformsToClean {
client, err := getClientForPlatform(ctx, buildersMap, builderImage, builderConfigPath, platform)
client, err := getClientForPlatform(ctx, buildersMap, bc, platform)
if err != nil {
return fmt.Errorf("cannot get client: %s", err)
}
@@ -143,12 +143,12 @@ func DiskUsage(buildersMap map[string]string, builderImage, builderConfigPath st
}
// PruneBuilder clean build cache of builder
func PruneBuilder(buildersMap map[string]string, builderImage, builderConfigPath string, platformsToClean []string, verbose bool) error {
func PruneBuilder(buildersMap map[string]string, bc BuilderConfig, platformsToClean []string, verbose bool) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
total := int64(0)
for _, platform := range platformsToClean {
client, err := getClientForPlatform(ctx, buildersMap, builderImage, builderConfigPath, platform)
client, err := getClientForPlatform(ctx, buildersMap, bc, platform)
if err != nil {
return fmt.Errorf("cannot get client: %s", err)
}