github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-08-22 16:47:24 +00:00
Code Issues Projects Releases Wiki Activity

option to pull down required images from to the cache, so that buildkit never gets them over the network (#4149)

Browse Source 
Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
Avi Deitcher 2025-07-27 18:07:20 +02:00 committed by GitHub
parent ef68e7bcd5
commit 1d3a8235a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 49 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cmd/linuxkit/pkg_build.go
View File

@ -43,6 +43,7 @@ func pkgBuildCmd() *cobra.Command {
builderImage string
builderConfig string
builderRestart bool
preCacheImages bool
release string
nobuild bool
manifest bool
@ -81,6 +82,9 @@ func pkgBuildCmd() *cobra.Command {
if ignoreCache {
opts = append(opts, pkglib.WithBuildIgnoreCache())
}
if preCacheImages {
opts = append(opts, pkglib.WithPreCacheImages())
}
if pull {
opts = append(opts, pkglib.WithBuildPull())
}
@ -283,6 +287,7 @@ func pkgBuildCmd() *cobra.Command {
cmd.Flags().StringVar(&builderImage, "builder-image", defaultBuilderImage, "buildkit builder container image to use")
cmd.Flags().StringVar(&builderConfig, "builder-config", "", "path to buildkit builder config.toml file to use, overrides the default config.toml in the builder image. When provided, copied over into builder, along with all certs. Use paths for certificates relative to your local host, they will be adjusted on copying into the container. USE WITH CAUTION")
cmd.Flags().BoolVar(&builderRestart, "builder-restart", false, "force restarting builder, even if container with correct name and image exists")
cmd.Flags().BoolVar(&preCacheImages, "precache-images", false, "download all referenced images in the Dockerfile to the linuxkit cache before building, thus referencing the local cache instead of pulling from the registry; this is useful for handling mirrors and special connections")
cmd.Flags().Var(&cacheDir, "cache", fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))
cmd.Flags().StringVar(&release, "release", "", "Release the given version")
cmd.Flags().BoolVar(&nobuild, "nobuild", false, "Skip building the image before pushing, conflicts with -force")

11
src/cmd/linuxkit/pkglib/build.go
View File

@ -29,6 +29,7 @@ type buildOpts struct {
pull bool
ignoreCache bool
push bool
preCacheImages bool
release string
manifest bool
targetDocker bool
@ -190,6 +191,14 @@ func WithBuildIgnoreCache() BuildOpt {
}
}
// WithPreCacheImages when building an image, download all referenced images in the Dockerfile to the linuxkit cache before building
func WithPreCacheImages() BuildOpt {
return func(bo *buildOpts) error {
bo.preCacheImages = true
return nil
}
}
// WithBuildSbomScanner when building an image, scan using the provided scanner image; if blank, uses the default
func WithBuildSbomScanner(scanner string) BuildOpt {
return func(bo *buildOpts) error {
@ -690,7 +699,7 @@ func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c spec.CacheProvider
imageBuildOpts.Dockerfile = bo.dockerfile
if err := d.build(ctx, tagArch, p.path, builderName, builderImage, builderConfigPath, platform, restart, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
if err := d.build(ctx, tagArch, p.path, builderName, builderImage, builderConfigPath, platform, restart, bo.preCacheImages, passCache, buildCtx.Reader(), stdout, bo.sbomScan, bo.sbomScannerImage, bo.progress, imageBuildOpts); err != nil {
stdoutCloser()
if strings.Contains(err.Error(), "executor failed running [/dev/.buildkit_qemu_emulator") {
return nil, fmt.Errorf("buildkit was unable to emulate %s. check binfmt has been set up and works for this platform: %v", platform, err)

2
src/cmd/linuxkit/pkglib/build_test.go
View File

@ -56,7 +56,7 @@ func (d *dockerMocker) contextSupportCheck() error {
func (d *dockerMocker) builder(_ context.Context, _, _, _, _ string, _ bool) (*buildkitClient.Client, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, builderRestart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, builderRestart, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progress string, imageBuildOpts spec.ImageBuildOptions) error {
if !d.enableBuild {
return errors.New("build disabled")
}

39
src/cmd/linuxkit/pkglib/docker.go
View File

@ -37,6 +37,7 @@ import (
"github.com/moby/buildkit/frontend/dockerui"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/progress/progressui"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
// golint requires comments on non-main(test)
// package for blank import
@ -67,7 +68,7 @@ const (
type dockerRunner interface {
tag(ref, tag string) error
build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart, preCacheImages bool, c spec.CacheProvider, r io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, platformType string, imageBuildOpts spec.ImageBuildOptions) error
save(tgt string, refs ...string) error
load(src io.Reader) error
pull(img string) (bool, error)
@ -511,7 +512,7 @@ func (dr *dockerRunnerImpl) tag(ref, tag string) error {
return dr.command(nil, nil, nil, "image", "tag", ref, tag)
}
func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, builderConfigPath, platform string, restart, preCacheImages bool, c spec.CacheProvider, stdin io.Reader, stdout io.Writer, sbomScan bool, sbomScannerImage, progressType string, imageBuildOpts spec.ImageBuildOptions) error {
// ensure we have a builder
client, err := dr.builder(ctx, dockerContext, builderImage, builderConfigPath, platform, restart)
if err != nil {
@ -632,6 +633,7 @@ func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext,
frontendAttrs["filename"] = imageBuildOpts.Dockerfile
// go through the dockerfile to see if we have any provided images cached
// and if we should cache any
if c != nil {
dockerfileRef := path.Join(pkg, imageBuildOpts.Dockerfile)
f, err := os.Open(dockerfileRef)
@ -692,12 +694,37 @@ func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext,
if err != nil {
return fmt.Errorf("invalid name %s", name)
}
// not found, so nothing to look up
if gdesc == nil {
// 3 possibilities:
// 1. we found it, so we can use it
// 2. we did not find it, but we were told to pre-cache images, so we pull it down and then use it
// 3. we did not find it, and we were not told to pre-cache images, so we just skip it
switch {
case gdesc == nil && !preCacheImages:
log.Debugf("image %s not found in cache, buildkit will pull directly", name)
continue
case gdesc == nil && preCacheImages:
log.Debugf("image %s not found in cache, pulling to pre-cache", name)
parts := strings.SplitN(platform, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return fmt.Errorf("invalid platform %s, expected format os/arch", platform)
}
plats := []imagespec.Platform{{OS: parts[0], Architecture: parts[1]}}
if err := c.ImagePull(&ref, plats, false); err != nil {
return fmt.Errorf("unable to pull image %s for caching: %v", name, err)
}
gdesc2, err := c.FindDescriptor(&ref)
if err != nil {
return fmt.Errorf("invalid name %s", name)
}
if gdesc2 == nil {
return fmt.Errorf("image %s not found in cache after pulling", name)
}
imageStores[name] = gdesc2.Digest.String()
default:
log.Debugf("image %s found in cache", name)
imageStores[name] = gdesc.Digest.String()
}
hash := gdesc.Digest
imageStores[name] = hash.String()
}
if len(imageStores) > 0 {
// if we made it here, we found the reference