diff --git a/docs/packages.md b/docs/packages.md index a79fd2ff8..a617d602d 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -8,7 +8,7 @@ in a LinuxKit-based project, if you know how to build a container, you should be able to build a LinuxKit package. All official LinuxKit packages are: -- Enabled with multi-arch manifests to work on multiple architectures. +- Enabled with multi-arch indexes to work on multiple architectures. - Derived from well-known sources for repeatable builds. - Built with multi-stage builds to minimise their size. @@ -67,7 +67,7 @@ A package source consists of a directory containing at least two files: ### Prerequisites Before you can build packages you need: -- Docker version 19.03 or newer, which includes [buildx](https://docs.docker.com/buildx/working-with-buildx/) +- Docker version 19.03 or newer. - If you are on a Mac you also need `docker-credential-osxkeychain.bin`, which comes with Docker for Mac. - `make`, `base64`, `jq`, and `expect` - A *recent* version of `manifest-tool` which you can build with `make @@ -147,14 +147,19 @@ a MacBook with Apple Silicon running on `arm64`. How does linuxkit determine where to build the target images? -linuxkit uses a combination of buildx builders and docker contexts, controlled by your input, to determine where to build. +linuxkit uses [buildkit](https://github.com/moby/buildkit) directly to build all images. +It uses docker contexts to determine _where_ to run those buildkit containers, based on the target +architecture. -Upon startup, it looks for a buildx builder named `linuxkit`. If it cannot find that builder, it creates it. +When running a package build, linuxkit looks for a container named `linuxkit-builder`, running the appropriate +version of buildkit. If it cannot find a container with that name, it creates it. +If the container already exists but is not running buildkit, or if the version is incorrect, linuxkit stops and removes +the existing `linuxkit-builder` container and creates one running the correct version of buildkit. When linuxkit needs to build a package for a particular architecture: -1. If a context for that architecture was provided, use that context. -1. If no context for that architecture was provided, use the default `linuxkit` context +1. If a context for that architecture was provided, use that context, looking for and/or starting a buildkit container named `linuxkit-builder`. +1. If no context for that architecture was provided, use the `default` context. The actual building then will be one of: @@ -183,14 +188,14 @@ linuxkit is capable of using native build nodes to do the build, even remotely. 1. Create a [docker context](https://docs.docker.com/engine/context/working-with-contexts/) that references the build node 1. Tell linuxkit to use that context for that architecture -linuxkit will then use that provided context to create a buildx builder and use it for that architecture. +linuxkit will then use that provided context to look for and/or start a container in which to run buildkit for that architecture. linuxkit looks for contexts in the following descending order of priority: 1. CLI option `--builders =,=`, e.g. `--builders linux/arm64=linuxkit-arm64,linux/amd64=default` 1. Environment variable `LINUXKIT_BUILDERS==,=`, e.g. `LINUXKIT_BUILDERS=linux/arm64=linuxkit-arm64,linux/amd64=default` 1. Existing context named `linuxkit-`, e.g. `linuxkit-linux-arm64` or `linuxkit-linux-s390x`, with "/" replaced by "-", as "/" is an invalid character. -1. Default builder named `linuxkit`, created by linuxkit, running in the default context +1. Default context If a builder name is provided for a specific platform, and it doesn't exist, it will be treated as a fatal error. @@ -200,10 +205,9 @@ If a builder name is provided for a specific platform, and it doesn't exist, it There are no contexts starting with `linuxkit-`, no environment variable `LINUXKIT_BUILDERS`, no command-line argument `--builders`. -linuxkit will build any requested packages using `docker buildx` on the local platform, with a builder (created, if necessary) named `linuxkit`. +linuxkit will build any requested packages using `default` context on the local platform, with a container (created, if necessary) named `linuxkit-builder`. Builds for the same architecture will be native, builds for other platforms will use either qemu or cross-building. - ##### Specified target You create a context named `my-remote-arm64` and then run: @@ -215,10 +219,13 @@ linuxkit pkg build --platforms=linux/arm64,linux/amd64 --builders linux/arm64=my linuxkit will build: * for arm64 using the context `my-remote-arm64`, since you specified in `--builders` to use `my-remote-arm64` for `linux/arm64` -* for amd64 using the context `default` and the `linuxkit` builder, as that is the default fallback +* for amd64 using the context `default`, as that is the default fallback The same would happen if you used `LINUXKIT_BUILDERS=linux/arm64=my-remote-arm64` instead of the `--builders` flag. +In both cases - the remote context `my-remote-arm64` and the local `default` context - it will do the build inside +a container named `linuxkit-builder`. + ##### Named context You create a context named `linuxkit-linux-arm64` and then run: @@ -234,7 +241,7 @@ linuxkit will build: ##### Combination -You create a context named `linuxkit-arm64`, and another named `my-remote-builder-amd64` and then run: +You create a context named `linuxkit-linux-arm64`, and another named `my-remote-builder-amd64` and then run: ```bash linuxkit pkg build --platforms=linux/arm64,linux/amd64 --builders linux/amd64=my-remote-builder-amd64 @@ -242,7 +249,7 @@ linuxkit pkg build --platforms=linux/arm64,linux/amd64 --builders linux/amd64=my linuxkit will build: -* for arm64 using the context `linuxkit-arm64`, since there is a context with the name `linuxkit-`, and you did not override that particular architecture using `--builders` or the environment variable `LINUXKIT_BUILDERS` +* for arm64 using the context `linuxkit-linux-arm64`, since there is a context with the name `linuxkit-`, and you did not override that particular architecture using `--builders` or the environment variable `LINUXKIT_BUILDERS` * for amd64 using the context `my-remote-builder-amd64`, since you specified for that architecture using `--builders` The same would happen if you used `LINUXKIT_BUILDERS=linux/arm64=my-remote-builder-amd64` instead of the `--builders` flag. diff --git a/src/cmd/linuxkit/gcp.go b/src/cmd/linuxkit/gcp.go index ef2238f64..f97a9746b 100644 --- a/src/cmd/linuxkit/gcp.go +++ b/src/cmd/linuxkit/gcp.go @@ -10,7 +10,7 @@ import ( "os" "time" - "github.com/docker/docker/pkg/term" + "github.com/moby/term" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/net/context" diff --git a/src/cmd/linuxkit/pkg_build.go b/src/cmd/linuxkit/pkg_build.go index ba8525a18..432361e97 100644 --- a/src/cmd/linuxkit/pkg_build.go +++ b/src/cmd/linuxkit/pkg_build.go @@ -12,7 +12,8 @@ import ( ) const ( - buildersEnvVar = "LINUXKIT_BUILDERS" + buildersEnvVar = "LINUXKIT_BUILDERS" + defaultBuilderImage = "moby/buildkit:v0.10.3" ) func pkgBuild(args []string) { @@ -38,6 +39,8 @@ func pkgBuildPush(args []string, withPush bool) { platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built") skipPlatforms := flags.String("skip-platforms", "", "Platforms that should be skipped, even if present in build.yml") builders := flags.String("builders", "", "Which builders to use for which platforms, e.g. linux/arm64=docker-context-arm64, overrides defaults and environment variables, see https://github.com/linuxkit/linuxkit/blob/master/docs/packages.md#Providing-native-builder-nodes") + builderImage := flags.String("builder-image", defaultBuilderImage, "buildkit builder container image to use") + builderRestart := flags.Bool("builder-restart", false, "force restarting builder, even if container with correct name and image exists") buildCacheDir := flags.String("cache", defaultLinuxkitCache(), "Directory for storing built image, incompatible with --docker") // some logic clarification: @@ -140,6 +143,8 @@ func pkgBuildPush(args []string, withPush bool) { os.Exit(1) } opts = append(opts, pkglib.WithBuildBuilders(buildersMap)) + opts = append(opts, pkglib.WithBuildBuilderImage(*builderImage)) + opts = append(opts, pkglib.WithBuildBuilderRestart(*builderRestart)) for _, p := range pkgs { // things we need our own copies of diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index 9ae91dd84..18b7ebe00 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -2,6 +2,7 @@ package pkglib import ( "archive/tar" + "context" "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( "strings" "github.com/containerd/containerd/reference" + "github.com/docker/docker/api/types" registry "github.com/google/go-containerregistry/pkg/v1" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/cache" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" @@ -27,19 +29,21 @@ const ( ) type buildOpts struct { - skipBuild bool - force bool - push bool - release string - manifest bool - image bool - targetDocker bool - cacheDir string - cacheProvider lktspec.CacheProvider - platforms []imagespec.Platform - builders map[string]string - runner dockerRunner - writer io.Writer + skipBuild bool + force bool + push bool + release string + manifest bool + image bool + targetDocker bool + cacheDir string + cacheProvider lktspec.CacheProvider + platforms []imagespec.Platform + builders map[string]string + runner dockerRunner + writer io.Writer + builderImage string + builderRestart bool } // BuildOpt allows callers to specify options to Build @@ -141,9 +145,26 @@ func WithBuildOutputWriter(w io.Writer) BuildOpt { } } +// WithBuildBuilderImage set the builder container image to use. +func WithBuildBuilderImage(image string) BuildOpt { + return func(bo *buildOpts) error { + bo.builderImage = image + return nil + } +} + +// WithBuildBuilderRestart restart the builder container even if it already is running with the correct image version +func WithBuildBuilderRestart(restart bool) BuildOpt { + return func(bo *buildOpts) error { + bo.builderRestart = restart + return nil + } +} + // Build builds the package func (p Pkg) Build(bos ...BuildOpt) error { var bo buildOpts + var ctx = context.TODO() for _, fn := range bos { if err := fn(&bo); err != nil { return err @@ -209,8 +230,8 @@ func (p Pkg) Build(bos ...BuildOpt) error { } } - if err := d.buildkitCheck(); err != nil { - return fmt.Errorf("buildkit not supported, check docker version: %v", err) + if err := d.contextSupportCheck(); err != nil { + return fmt.Errorf("contexts not supported, check docker version: %v", err) } skipBuild := bo.skipBuild @@ -227,23 +248,31 @@ func (p Pkg) Build(bos ...BuildOpt) error { if !skipBuild { fmt.Fprintf(writer, "building %s\n", ref) var ( - args []string + imageBuildOpts = types.ImageBuildOptions{ + Labels: map[string]string{}, + BuildArgs: map[string]*string{}, + } descs []registry.Descriptor ) + // args that we use: + // labels map[string]string + // network string + // build-arg []string + if p.git != nil && p.gitRepo != "" { - args = append(args, "--label", "org.opencontainers.image.source="+p.gitRepo) + imageBuildOpts.Labels["org.opencontainers.image.source"] = p.gitRepo } if p.git != nil && !p.dirty { commit, err := p.git.commitHash("HEAD") if err != nil { return err } - args = append(args, "--label", "org.opencontainers.image.revision="+commit) + imageBuildOpts.Labels["org.opencontainers.image.revision"] = commit } if !p.network { - args = append(args, "--network=none") + imageBuildOpts.NetworkMode = "none" } if p.config != nil { @@ -251,21 +280,25 @@ func (p Pkg) Build(bos ...BuildOpt) error { if err != nil { return err } - args = append(args, "--label=org.mobyproject.config="+string(b)) + imageBuildOpts.Labels["org.mobyproject.config"] = string(b) } - args = append(args, "--label=org.mobyproject.linuxkit.version="+version.Version) - args = append(args, "--label=org.mobyproject.linuxkit.revision="+version.GitCommit) + imageBuildOpts.Labels["org.mobyproject.linuxkit.version"] = version.Version + imageBuildOpts.Labels["org.mobyproject.linuxkit.revision"] = version.GitCommit if p.buildArgs != nil { for _, buildArg := range *p.buildArgs { - args = append(args, "--build-arg", buildArg) + parts := strings.SplitN(buildArg, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid build-arg, must be in format 'arg=value': %s", buildArg) + } + imageBuildOpts.BuildArgs[parts[0]] = &parts[1] } } // build for each arch and save in the linuxkit cache for _, platform := range bo.platforms { - desc, err := p.buildArch(d, c, platform.Architecture, args, writer, bo) + desc, err := p.buildArch(ctx, d, c, bo.builderImage, platform.Architecture, bo.builderRestart, writer, bo, imageBuildOpts) if err != nil { return fmt.Errorf("error building for arch %s: %v", platform.Architecture, err) } @@ -354,7 +387,7 @@ func (p Pkg) Build(bos ...BuildOpt) error { } // buildArch builds the package for a single arch -func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, args []string, writer io.Writer, bo buildOpts) (*registry.Descriptor, error) { +func (p Pkg) buildArch(ctx context.Context, d dockerRunner, c lktspec.CacheProvider, builderImage, arch string, restart bool, writer io.Writer, bo buildOpts, imageBuildOpts types.ImageBuildOptions) (*registry.Descriptor, error) { var ( desc *registry.Descriptor tagArch string @@ -388,7 +421,6 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg // set the target var ( - buildxOutput string stdout io.WriteCloser eg errgroup.Group stdoutCloser = func() { @@ -403,7 +435,6 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg } // we are writing to local, so we need to catch the tar output stream and place the right files in the right place - buildxOutput = "type=oci" piper, pipew := io.Pipe() stdout = pipew @@ -418,13 +449,10 @@ func (p Pkg) buildArch(d dockerRunner, c lktspec.CacheProvider, arch string, arg piper.Close() return err }) - args = append(args, fmt.Sprintf("--output=%s", buildxOutput)) buildCtx := &buildCtx{sources: p.sources} platform := fmt.Sprintf("linux/%s", arch) - archArgs := append(args, "--platform") - archArgs = append(archArgs, platform) - if err := d.build(tagArch, p.path, builderName, platform, buildCtx.Reader(), stdout, archArgs...); err != nil { + if err := d.build(ctx, tagArch, p.path, builderName, builderImage, platform, restart, buildCtx.Reader(), stdout, imageBuildOpts); err != nil { stdoutCloser() if strings.Contains(err.Error(), "executor failed running [/dev/.buildkit_qemu_emulator") { return nil, fmt.Errorf("buildkit was unable to emulate %s. check binfmt has been set up and works for this platform: %v", platform, err) diff --git a/src/cmd/linuxkit/pkglib/build_test.go b/src/cmd/linuxkit/pkglib/build_test.go index 708f832df..eaa5cc714 100644 --- a/src/cmd/linuxkit/pkglib/build_test.go +++ b/src/cmd/linuxkit/pkglib/build_test.go @@ -2,6 +2,7 @@ package pkglib import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "testing" "github.com/containerd/containerd/reference" + dockertypes "github.com/docker/docker/api/types" registry "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" lktspec "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" @@ -20,7 +22,7 @@ import ( ) type dockerMocker struct { - supportBuildKit bool + supportContexts bool images map[string][]byte enableTag bool enableBuild bool @@ -34,15 +36,8 @@ type buildLog struct { pkg string dockerContext string platform string - opts []string } -func (d *dockerMocker) buildkitCheck() error { - if d.supportBuildKit { - return nil - } - return errors.New("buildkit unsupported") -} func (d *dockerMocker) tag(ref, tag string) error { if !d.enableTag { return errors.New("tags not allowed") @@ -50,11 +45,18 @@ func (d *dockerMocker) tag(ref, tag string) error { d.images[tag] = d.images[ref] return nil } -func (d *dockerMocker) build(tag, pkg, dockerContext, platform string, stdin io.Reader, stdout io.Writer, opts ...string) error { +func (d *dockerMocker) contextSupportCheck() error { + if d.supportContexts { + return nil + } + return errors.New("contexts not supported") +} + +func (d *dockerMocker) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, builderRestart bool, stdin io.Reader, stdout io.Writer, imageBuildOpts dockertypes.ImageBuildOptions) error { if !d.enableBuild { return errors.New("build disabled") } - d.builds = append(d.builds, buildLog{tag, pkg, dockerContext, platform, opts}) + d.builds = append(d.builds, buildLog{tag, pkg, dockerContext, platform}) return nil } func (d *dockerMocker) save(tgt string, refs ...string) error { @@ -297,13 +299,13 @@ func TestBuild(t *testing.T) { err string }{ {"invalid tag", Pkg{image: "docker.io/foo/bar:abc:def:ghi"}, nil, nil, &dockerMocker{}, &cacheMocker{}, "could not resolve references"}, - {"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"}, - {"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must provide linuxkit build cache"}, - {"unsupported buildkit", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "buildkit not supported, check docker version"}, - {"load docker without local platform", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir), WithBuildTargetDockerCache()}, []string{nonLocal}, &dockerMocker{supportBuildKit: false}, &cacheMocker{}, "must build for local platform"}, - {"amd64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportBuildKit: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, - {"arm64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"arm64"}, &dockerMocker{supportBuildKit: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, - {"amd64 and arm64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64", "arm64"}, &dockerMocker{supportBuildKit: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, + {"not at head", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "foo"}, nil, []string{"amd64"}, &dockerMocker{supportContexts: false}, &cacheMocker{}, "Cannot build from commit hash != HEAD"}, + {"no build cache", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, nil, []string{"amd64"}, &dockerMocker{supportContexts: false}, &cacheMocker{}, "must provide linuxkit build cache"}, + {"unsupported contexts", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportContexts: false}, &cacheMocker{}, "contexts not supported, check docker version"}, + {"load docker without local platform", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir), WithBuildTargetDockerCache()}, []string{nonLocal}, &dockerMocker{supportContexts: false}, &cacheMocker{}, "must build for local platform"}, + {"amd64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64"}, &dockerMocker{supportContexts: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, + {"arm64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"arm64"}, &dockerMocker{supportContexts: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, + {"amd64 and arm64", Pkg{org: "foo", image: "bar", hash: "abc", arches: []string{"amd64", "arm64"}, commitHash: "HEAD"}, []BuildOpt{WithBuildCacheDir(cacheDir)}, []string{"amd64", "arm64"}, &dockerMocker{supportContexts: true, enableBuild: true}, &cacheMocker{enableImagePull: false, enableImageLoad: true, enableIndexWrite: true}, ""}, } for _, tt := range tests { t.Run(tt.msg, func(t *testing.T) { @@ -324,7 +326,12 @@ func TestBuild(t *testing.T) { // need to make sure that it was called the correct number of times with the correct arguments t.Errorf("mismatched call to runners, should be %d was %d: %#v", len(tt.targets), len(tt.runner.builds), tt.runner.builds) case tt.err == "": - // check that all of our platforms were called + // check that all of our platforms were called exactly once each + // we do that by: + // 1- creating a map of all of the target platforms and setting them to `false` + // 2- checking with each build for which platform it was called + // + // each build is assumed to track what platform it built platformMap := map[string]bool{} for _, arch := range tt.targets { platformMap[fmt.Sprintf("linux/%s", arch)] = false @@ -334,6 +341,11 @@ func TestBuild(t *testing.T) { t.Errorf("mismatch in build: '%v', %#v", err, build) } } + for k, v := range platformMap { + if !v { + t.Errorf("did not execute build for platform: %s", k) + } + } } }) } @@ -341,23 +353,14 @@ func TestBuild(t *testing.T) { // testCheckBuildRun check the output of a build run func testCheckBuildRun(build buildLog, platforms map[string]bool) error { - for i, arg := range build.opts { - switch { - case arg == "--platform", arg == "-platform": - if i+1 >= len(build.opts) { - return errors.New("provided arg --platform with no next argument") - } - platform := build.opts[i+1] - used, ok := platforms[platform] - if !ok { - return fmt.Errorf("requested unknown platform: %s", platform) - } - if used { - return fmt.Errorf("tried to use platform twice: %s", platform) - } - platforms[platform] = true - return nil - } + platform := build.platform + used, ok := platforms[platform] + if !ok { + return fmt.Errorf("requested unknown platform: %s", platform) } - return errors.New("missing platform argument") + if used { + return fmt.Errorf("tried to use platform twice: %s", platform) + } + platforms[platform] = true + return nil } diff --git a/src/cmd/linuxkit/pkglib/docker.go b/src/cmd/linuxkit/pkglib/docker.go index dcaccc418..0cb9981c6 100644 --- a/src/cmd/linuxkit/pkglib/docker.go +++ b/src/cmd/linuxkit/pkglib/docker.go @@ -7,6 +7,7 @@ package pkglib import ( "bufio" "bytes" + "context" "encoding/json" "errors" "fmt" @@ -16,24 +17,35 @@ import ( "os/exec" "runtime" "strings" + "time" + "github.com/docker/buildx/util/progress" + "github.com/docker/docker/api/types" versioncompare "github.com/hashicorp/go-version" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/registry" + buildkitClient "github.com/moby/buildkit/client" + _ "github.com/moby/buildkit/client/connhelper/dockercontainer" + _ "github.com/moby/buildkit/client/connhelper/ssh" + "github.com/moby/buildkit/frontend/dockerfile/builder" + "github.com/moby/buildkit/session/upload/uploadprovider" log "github.com/sirupsen/logrus" ) const ( - registryServer = "https://index.docker.io/v1/" - buildkitBuilderName = "linuxkit" + registryServer = "https://index.docker.io/v1/" + buildkitBuilderName = "linuxkit-builder" + buildkitSocketPath = "/run/buildkit/buildkitd.sock" + buildkitWaitServer = 30 // seconds + buildkitCheckInterval = 1 // seconds ) type dockerRunner interface { - buildkitCheck() error tag(ref, tag string) error - build(tag, pkg, dockerContext, platform string, stdin io.Reader, stdout io.Writer, opts ...string) error + build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, stdin io.Reader, stdout io.Writer, imageBuildOpts types.ImageBuildOptions) error save(tgt string, refs ...string) error load(src io.Reader) error pull(img string) (bool, error) + contextSupportCheck() error } type dockerRunnerImpl struct { @@ -171,101 +183,134 @@ func (dr *dockerRunnerImpl) versionCheck(version string) (string, string, error) return clientVersionString, serverVersionString, nil } -// buildkitCheck checks if buildkit is supported. This is necessary because github uses some strange versions -// of docker in Actions, which makes it difficult to tell if buildkit is supported. +// contextCheck checks if contexts are supported. This is necessary because github uses some strange versions +// of docker in Actions, which makes it difficult to tell if context is supported. // See https://github.community/t/what-really-is-docker-3-0-6/16171 -func (dr *dockerRunnerImpl) buildkitCheck() error { - return dr.command(nil, ioutil.Discard, ioutil.Discard, "buildx", "ls") +func (dr *dockerRunnerImpl) contextSupportCheck() error { + return dr.command(nil, ioutil.Discard, ioutil.Discard, "context", "ls") } -// builder ensure that a builder exists. Works as follows. +// builder ensure that a builder container exists or return an error. +// +// Process: +// +// 1. Get an appropriate docker context. +// 2. Using the appropriate context, try to find a docker container named `linuxkit-builder` in that context. +// 3. Return a reference to that container. +// +// To get the appropriate docker context: +// // 1. if dockerContext is provided, try to create a builder with that context; if it succeeds, we are done; if not, return an error. // 2. try to find an existing named runner with the pattern; if it succeeds, we are done; if not, try next. // 3. try to create a generic builder using the default context named "linuxkit". -func (dr *dockerRunnerImpl) builder(dockerContext, platform string) (string, error) { - var ( - builderName string - args = []string{"buildx", "create", "--driver", "docker-container", "--buildkitd-flags", "--allow-insecure-entitlement network.host"} - ) - +func (dr *dockerRunnerImpl) builder(ctx context.Context, dockerContext, builderImage, platform string, restart bool) (*buildkitClient.Client, error) { // if we were given a context, we must find a builder and use it, or create one and use it if dockerContext != "" { // does the context exist? if err := dr.command(nil, ioutil.Discard, ioutil.Discard, "context", "inspect", dockerContext); err != nil { - return "", fmt.Errorf("provided docker context '%s' not found", dockerContext) + return nil, fmt.Errorf("provided docker context '%s' not found", dockerContext) } - builderName = fmt.Sprintf("%s-%s-%s-builder", buildkitBuilderName, dockerContext, strings.ReplaceAll(platform, "/", "-")) - if err := dr.builderEnsureContainer(builderName, platform, dockerContext, args...); err != nil { - return "", fmt.Errorf("error preparing builder based on context '%s': %v", dockerContext, err) + client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, platform, dockerContext, restart) + if err != nil { + return nil, fmt.Errorf("error preparing builder based on context '%s': %v", dockerContext, err) } - return builderName, nil + return client, nil } // no provided dockerContext, so look for one based on platform-specific name - dockerContext = fmt.Sprintf("%s-%s", buildkitBuilderName, strings.ReplaceAll(platform, "/", "-")) + dockerContext = fmt.Sprintf("%s-%s", "linuxkit", strings.ReplaceAll(platform, "/", "-")) if err := dr.command(nil, ioutil.Discard, ioutil.Discard, "context", "inspect", dockerContext); err == nil { // we found an appropriately named context, so let us try to use it or error out - builderName = fmt.Sprintf("%s-builder", dockerContext) - if err := dr.builderEnsureContainer(builderName, platform, dockerContext, args...); err == nil { - return builderName, nil + if client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, platform, dockerContext, restart); err == nil { + return client, nil } } // create a generic builder - builderName = buildkitBuilderName - if err := dr.builderEnsureContainer(builderName, "", "", args...); err != nil { - return "", fmt.Errorf("error ensuring default builder '%s': %v", builderName, err) + client, err := dr.builderEnsureContainer(ctx, buildkitBuilderName, builderImage, "", "default", restart) + if err != nil { + return nil, fmt.Errorf("error ensuring builder container in default context: %v", err) } - return builderName, nil + return client, nil } -// builderEnsureContainer provided a name of a builder, ensure that the builder exists, and if not, create it -// based on the provided docker context, for the target platform.. Assumes the dockerContext already exists. -func (dr *dockerRunnerImpl) builderEnsureContainer(name, platform, dockerContext string, args ...string) error { +// builderEnsureContainer provided a name of a docker context, ensure that the builder container exists and +// is running the appropriate version of buildkit. If it does not exist, create it; if it is running +// but has the wrong version of buildkit, or not running buildkit at all, remove it and create an appropriate +// one. +// Returns a network connection to the buildkit builder in the container. +func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, image, platform, dockerContext string, forceRestart bool) (*buildkitClient.Client, error) { // if no error, then we have a builder already // inspect it to make sure it is of the right type var b bytes.Buffer - if err := dr.command(nil, &b, ioutil.Discard, "buildx", "inspect", name); err != nil { - // we did not have the named builder, so create the builder - args = append(args, "--name", name) - msg := fmt.Sprintf("creating builder '%s'", name) - if platform != "" { - args = append(args, "--platform", platform) - msg = fmt.Sprintf("%s for platform '%s'", msg, platform) - } else { - msg = fmt.Sprintf("%s for all supported platforms", msg) + if err := dr.command(nil, &b, ioutil.Discard, "--context", dockerContext, "container", "inspect", name); err == nil { + // we already have a container named "linuxkit-builder" in the provided context. + var restart bool + // get its state and config + var containerJSON []types.ContainerJSON + if err := json.Unmarshal(b.Bytes(), &containerJSON); err != nil || len(containerJSON) < 1 { + return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err) } - if dockerContext != "" { - args = append(args, dockerContext) - msg = fmt.Sprintf("%s based on docker context '%s'", msg, dockerContext) - } - fmt.Println(msg) - return dr.command(nil, ioutil.Discard, ioutil.Discard, args...) - } - // if we got here, we found a builder already, so let us check its type - var ( - scanner = bufio.NewScanner(&b) - driver string - ) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) < 2 { - continue - } - if fields[0] != "Driver:" { - continue - } - driver = fields[1] - break - } - switch driver { - case "": - return fmt.Errorf("builder '%s' exists but has no driver type", name) - case "docker-container": - return nil - default: - return fmt.Errorf("builder '%s' exists but has wrong driver type '%s'", name, driver) + existingImage := containerJSON[0].Config.Image + + switch { + case forceRestart: + // if restart==true, we always restart, else we check if it matches our requirements + fmt.Printf("told to force restart, replacing existing container %s\n", name) + restart = true + case existingImage != image: + // if image mismatches, restart + fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image) + restart = true + case !containerJSON[0].HostConfig.Privileged: + fmt.Printf("existing container %s is unprivileged, replacing\n", name) + restart = true + } + if !restart { + fmt.Printf("using existing container %s\n", name) + return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext)) + } + + // if we made it here, we need to stop and remove the container, either because of a config mismatch, + // or because we received the CLI option + if containerJSON[0].State.Status == "running" { + if err := dr.command(nil, ioutil.Discard, ioutil.Discard, "--context", dockerContext, "container", "stop", name); err != nil { + return nil, fmt.Errorf("failed to stop existing container %s", name) + } + } + if err := dr.command(nil, ioutil.Discard, ioutil.Discard, "--context", dockerContext, "container", "rm", name); err != nil { + return nil, fmt.Errorf("failed to remove existing container %s", name) + } + } + // create the builder + args := []string{"container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"} + msg := fmt.Sprintf("creating builder container '%s' in context '%s", name, dockerContext) + fmt.Println(msg) + if err := dr.command(nil, ioutil.Discard, ioutil.Discard, args...); err != nil { + return nil, err + } + // wait for buildkit socket to be ready up to the timeout + fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer) + timeout := time.After(buildkitWaitServer * time.Second) + ticker := time.Tick(buildkitCheckInterval * time.Second) + // Keep trying until we're timed out or get a success + for { + select { + // Got a timeout! fail with a timeout error + case <-timeout: + return nil, fmt.Errorf("could not communicate with buildkit builder at context/container %s/%s after %d seconds", dockerContext, name, buildkitWaitServer) + // Got a tick, we should try again + case <-ticker: + client, err := buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext)) + if err == nil { + fmt.Println("buildkit builder ready!") + return client, nil + } + + // got an error, wait 1 second and try again + log.Debugf("buildkitclient error: %v, waiting %d seconds and trying again", err, buildkitCheckInterval) + } } } @@ -319,37 +364,77 @@ func (dr *dockerRunnerImpl) tag(ref, tag string) error { return dr.command(nil, nil, nil, "image", "tag", ref, tag) } -func (dr *dockerRunnerImpl) build(tag, pkg, dockerContext, platform string, stdin io.Reader, stdout io.Writer, opts ...string) error { +func (dr *dockerRunnerImpl) build(ctx context.Context, tag, pkg, dockerContext, builderImage, platform string, restart bool, stdin io.Reader, stdout io.Writer, imageBuildOpts types.ImageBuildOptions) error { // ensure we have a builder - builderName, err := dr.builder(dockerContext, platform) + client, err := dr.builder(ctx, dockerContext, builderImage, platform, restart) if err != nil { - return fmt.Errorf("unable to ensure proper buildx builder: %v", err) + return fmt.Errorf("unable to ensure builder container: %v", err) } - args := []string{"buildx", "build"} + frontendAttrs := map[string]string{} for _, proxyVarName := range proxyEnvVars { if value, ok := os.LookupEnv(proxyVarName); ok { - args = append(args, - []string{"--build-arg", fmt.Sprintf("%s=%s", proxyVarName, value)}...) + frontendAttrs[proxyVarName] = value } } + // platform + frontendAttrs["platform"] = platform + + // build-args + for k, v := range imageBuildOpts.BuildArgs { + frontendAttrs[fmt.Sprintf("build-arg:%s", k)] = *v + } + + // no-cache option if !dr.cache { - args = append(args, "--no-cache") + frontendAttrs["no-cache"] = "" + } + + // network + frontendAttrs["network"] = imageBuildOpts.NetworkMode + + for k, v := range imageBuildOpts.Labels { + frontendAttrs[fmt.Sprintf("label:%s", k)] = v + } + + solveOpts := buildkitClient.SolveOpt{ + Frontend: "dockerfile.v0", + FrontendAttrs: frontendAttrs, + Exports: []buildkitClient.ExportEntry{ + { + Type: buildkitClient.ExporterOCI, + Attrs: map[string]string{ + "name": tag, + }, + Output: fixedWriteCloser(&writeNopCloser{stdout}), + }, + }, } - args = append(args, opts...) - args = append(args, fmt.Sprintf("--builder=%s", builderName)) - args = append(args, "-t", tag) - // should docker read from the build path or stdin? - buildPath := pkg if stdin != nil { - buildPath = "-" + buf := bufio.NewReader(stdin) + up := uploadprovider.New() + frontendAttrs["context"] = up.Add(buf) + solveOpts.Session = append(solveOpts.Session, up) + } else { + solveOpts.LocalDirs = map[string]string{ + builder.DefaultLocalNameDockerfile: pkg, + builder.DefaultLocalNameContext: pkg, + } } - args = append(args, buildPath) - fmt.Printf("building for platform %s using builder %s\n", platform, builderName) - return dr.command(stdin, stdout, nil, args...) + ctx2, cancel := context.WithCancel(context.TODO()) + defer cancel() + printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, "auto") + pw := progress.WithPrefix(printer, "", false) + ch, done := progress.NewChannel(pw) + defer func() { <-done }() + + fmt.Printf("building for platform %s\n", platform) + + _, err = client.Solve(ctx, nil, solveOpts, ch) + return err } func (dr *dockerRunnerImpl) save(tgt string, refs ...string) error { @@ -361,3 +446,20 @@ func (dr *dockerRunnerImpl) load(src io.Reader) error { args := []string{"image", "load"} return dr.command(src, nil, nil, args...) } + +func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { + return func(map[string]string) (io.WriteCloser, error) { + return wc, nil + } +} + +type writeNopCloser struct { + writer io.Writer +} + +func (w *writeNopCloser) Close() error { + return nil +} +func (w *writeNopCloser) Write(p []byte) (n int, err error) { + return w.writer.Write(p) +}