linuxkit: implement docker image dependencies for pkg build.

This allows the `linuxkit/kubernetes` "image-cache" packages to use a standard
`linuxkit pkg build` based flow rather than requiring surrounding scaffolding.

Fixes #2766. Compared with the original (actually, the second) proposal made in
issue #2766, the field is `docker-images` rather than `images` to allow for
future inclusion of e.g. `containerd-images`.

Signed-off-by: Ian Campbell <ijc@docker.com>

docs/packages.md

@@ -30,6 +30,10 @@ A package source consists of a directory containing at least two files:
 - `disable-content-trust` _(bool)_: Disable Docker content trust for this package (default: no)
 - `disable-cache` _(bool)_: Disable build cache for this package (default: no)
 - `config`: _(struct `github.com/moby/tool/src/moby.ImageConfig`)_: Image configuration, marshalled to JSON and added as `org.mobyproject.config` label on image (default: no label)
+- `depends`: Contains information on prerequisites which must be satisfied in order to build the package. Has subfields:
+    - `docker-images`: Docker images to be made available (as `tar` files via `docker image save`) within the package build context. Contains the following nested fields:
+        - `from-file` and `list`: _(string and string list respectively)_. Mutually exclusive fields specifying the list of images to include. Each image must include a valid digest (`sha256:...`) in order to maintain determinism. If `from-file` is used then it is a path relative to (and within) the package directory with one image per line (lines with `#` in column 0 and blank lines are ignore). If `list` is used then each entry is an image.
+        - `target` and `target-dir`: _(string)_ Mutually exclusive fields specifying the target location, if `target` is used then it is a path relative to (and within) the package dir which names a `tar` file into which all of the listed images will be saved. If `target-dir` then it is a path relative to (and within) the package directory which names a directory into which each image will be saved (as `«image name»@«digest».tar`). **NB**: The path referenced by `target-dir` will be _removed_ prior to populating (to avoid issues with stale files).
 
 ## Building packages
 

src/cmd/linuxkit/pkglib/build.go

@@ -110,6 +110,10 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 	if !bo.skipBuild {
 		var args []string
 
+		if err := p.dockerDepends.Do(d); err != nil {
+			return err
+		}
+
 		if p.git != nil && p.gitRepo != "" {
 			args = append(args, "--label", "org.opencontainers.image.source="+p.gitRepo)
 		}

src/cmd/linuxkit/pkglib/depends.go

@@ -0,0 +1,133 @@
+package pkglib
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/reference"
+)
+
+type dockerDepends struct {
+	images []reference.Spec
+	path   string
+	dir    bool
+}
+
+func newDockerDepends(pkgPath string, pi *pkgInfo) (dockerDepends, error) {
+	var err error
+
+	if (pi.Depends.DockerImages.TargetDir != "") && (pi.Depends.DockerImages.Target != "") {
+		return dockerDepends{}, fmt.Errorf("\"depends.images.target\" and \"depends.images.target-dir\" are mutually exclusive")
+	}
+	if (pi.Depends.DockerImages.FromFile != "") && (len(pi.Depends.DockerImages.List) > 0) {
+		return dockerDepends{}, fmt.Errorf("\"depends.images.list\" and \"depends.images.from-file\" are mutually exclusive")
+	}
+
+	if pi.Depends.DockerImages.Target, err = makeAbsSubpath("depends.image.target", pkgPath, pi.Depends.DockerImages.Target); err != nil {
+		return dockerDepends{}, err
+	}
+	if pi.Depends.DockerImages.TargetDir, err = makeAbsSubpath("depends.image.target-dir", pkgPath, pi.Depends.DockerImages.TargetDir); err != nil {
+		return dockerDepends{}, err
+	}
+	if pi.Depends.DockerImages.FromFile != "" {
+		p, err := makeAbsSubpath("depends.image.from-file", pkgPath, pi.Depends.DockerImages.FromFile)
+		if err != nil {
+			return dockerDepends{}, err
+		}
+		f, err := os.Open(p)
+		if err != nil {
+			return dockerDepends{}, err
+		}
+		defer f.Close()
+
+		s := bufio.NewScanner(f)
+		for s.Scan() {
+			t := s.Text()
+			if len(t) > 0 && t[0] != '#' {
+				pi.Depends.DockerImages.List = append(pi.Depends.DockerImages.List, s.Text())
+			}
+		}
+
+		if err := s.Err(); err != nil {
+			return dockerDepends{}, err
+		}
+	}
+
+	var specs []reference.Spec
+	for _, i := range pi.Depends.DockerImages.List {
+		s, err := reference.Parse(i)
+		if err != nil {
+			return dockerDepends{}, err
+		}
+		dgst := s.Digest()
+		if dgst == "" {
+			return dockerDepends{}, fmt.Errorf("image %q lacks a digest", i)
+		}
+		if err := dgst.Validate(); err != nil {
+			return dockerDepends{}, fmt.Errorf("unable to validate digest in %q: %v", i, err)
+		}
+		specs = append(specs, s)
+	}
+
+	var dir bool
+	path := pi.Depends.DockerImages.Target
+	if pi.Depends.DockerImages.TargetDir != "" {
+		path = pi.Depends.DockerImages.TargetDir
+		dir = true
+	}
+	return dockerDepends{
+		images: specs,
+		path:   path,
+		dir:    dir,
+	}, nil
+}
+
+// Do ensures that any dependencies the package has declared are met.
+func (dd dockerDepends) Do(d dockerRunner) error {
+	if len(dd.images) == 0 {
+		return nil
+	}
+
+	if dd.dir {
+		dir := dd.path
+
+		// Delete and recreate so it is empty
+		if err := os.RemoveAll(dir); err != nil {
+			return fmt.Errorf("failed to remove %q: %v", dir, err)
+		}
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create %q: %v", dir, err)
+		}
+	}
+
+	var refs []string
+
+	for _, s := range dd.images {
+		if ok, err := d.pull(s.String()); !ok || err != nil {
+			if err != nil {
+				return err
+			}
+			return fmt.Errorf("failed to pull %q", s.String())
+		}
+
+		refs = append(refs, s.Locator)
+		if dd.dir {
+			bn := filepath.Base(s.Locator) + "@" + s.Digest().String()
+			path := filepath.Join(dd.path, bn+".tar")
+			fmt.Printf("Adding %q as dependency\n", bn)
+			if err := d.save(path, s.String()); err != nil {
+				return err
+			}
+		}
+	}
+
+	if !dd.dir {
+		if err := d.save(dd.path, refs...); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

src/cmd/linuxkit/pkglib/docker.go

@@ -103,3 +103,8 @@ func (dr dockerRunner) build(tag, pkg string, opts ...string) error {
 	args = append(args, "-t", tag, pkg)
 	return dr.command(args...)
 }
+
+func (dr dockerRunner) save(tgt string, refs ...string) error {
+	args := append([]string{"image", "save", "-o", tgt}, refs...)
+	return dr.command(args...)
+}

src/cmd/linuxkit/pkglib/pkglib.go

@@ -12,7 +12,7 @@ import (
 	"github.com/moby/tool/src/moby"
 )
 
-// Containers fields settable in the build.yml
+// Contains fields settable in the build.yml
 type pkgInfo struct {
 	Image               string            `yaml:"image"`
 	Org                 string            `yaml:"org"`
@@ -22,19 +22,28 @@ type pkgInfo struct {
 	DisableContentTrust bool              `yaml:"disable-content-trust"`
 	DisableCache        bool              `yaml:"disable-cache"`
 	Config              *moby.ImageConfig `yaml:"config"`
+	Depends             struct {
+		DockerImages struct {
+			TargetDir string   `yaml:"target-dir"`
+			Target    string   `yaml:"target"`
+			FromFile  string   `yaml:"from-file"`
+			List      []string `yaml:"list"`
+		} `yaml:"docker-images"`
+	} `yaml:"depends"`
 }
 
 // Pkg encapsulates information about a package's source
 type Pkg struct {
 	// These correspond to pkgInfo fields
-	image   string
+	image         string
-	org     string
+	org           string
-	arches  []string
+	arches        []string
-	gitRepo string
+	gitRepo       string
-	network bool
+	network       bool
-	trust   bool
+	trust         bool
-	cache   bool
+	cache         bool
-	config  *moby.ImageConfig
+	config        *moby.ImageConfig
+	dockerDepends dockerDepends
 
 	// Internal state
 	path       string
@@ -123,6 +132,11 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 		return Pkg{}, fmt.Errorf("Image field is required")
 	}
 
+	dockerDepends, err := newDockerDepends(pkgPath, &pi)
+	if err != nil {
+		return Pkg{}, err
+	}
+
 	if devMode {
 		// If --org is also used then this will be overwritten
 		// by argOrg when we iterate over the provided options
@@ -180,19 +194,20 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 	}
 
 	return Pkg{
-		image:      pi.Image,
+		image:         pi.Image,
-		org:        pi.Org,
+		org:           pi.Org,
-		hash:       hash,
+		hash:          hash,
-		commitHash: hashCommit,
+		commitHash:    hashCommit,
-		arches:     pi.Arches,
+		arches:        pi.Arches,
-		gitRepo:    pi.GitRepo,
+		gitRepo:       pi.GitRepo,
-		network:    pi.Network,
+		network:       pi.Network,
-		trust:      !pi.DisableContentTrust,
+		trust:         !pi.DisableContentTrust,
-		cache:      !pi.DisableCache,
+		cache:         !pi.DisableCache,
-		config:     pi.Config,
+		config:        pi.Config,
-		dirty:      dirty,
+		dockerDepends: dockerDepends,
-		path:       pkgPath,
+		dirty:         dirty,
-		git:        git,
+		path:          pkgPath,
+		git:           git,
 	}, nil
 }
 
@@ -242,3 +257,29 @@ func (p Pkg) cleanForBuild() error {
 	}
 	return nil
 }
+
+// Expands path from relative to abs against base, ensuring the result is within base, but is not base itself. Field is the fieldname, to be used for constructing the error.
+func makeAbsSubpath(field, base, path string) (string, error) {
+	if path == "" {
+		return "", nil
+	}
+
+	if filepath.IsAbs(path) {
+		return "", fmt.Errorf("%s must be relative to package directory", field)
+	}
+
+	p, err := filepath.Abs(filepath.Join(base, path))
+	if err != nil {
+		return "", err
+	}
+
+	if p == base {
+		return "", fmt.Errorf("%s must not be exactly the package directory", field)
+	}
+
+	if !filepath.HasPrefix(p, base) {
+		return "", fmt.Errorf("%s must be within package directory", field)
+	}
+
+	return p, nil
+}

src/cmd/linuxkit/pkglib/pkglib_test.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -85,3 +86,63 @@ func TestCache(t *testing.T) {
 func TestContentTrust(t *testing.T) {
 	testBool(t, "disable-content-trust", true, "-enable-content-trust", "-disable-content-trust", func(p Pkg) bool { return p.trust })
 }
+
+func testBadBuildYML(t *testing.T, build, expect string) {
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	tmpDir := filepath.Join(cwd, t.Name())
+	err = os.Mkdir(tmpDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	pkgDir := dummyPackage(t, tmpDir, build)
+	flags := flag.NewFlagSet(t.Name(), flag.ExitOnError)
+	args := []string{"-hash-path=" + cwd, pkgDir}
+	_, err = NewFromCLI(flags, args...)
+	require.Error(t, err)
+	assert.Regexp(t, regexp.MustCompile(expect), err.Error())
+}
+
+func TestDependsImageNoDigest(t *testing.T) {
+	testBadBuildYML(t, `
+image: dummy
+depends:
+  docker-images:
+    target-dir: dl
+    list:
+      - docker.io/library/nginx:latest
+`, `image ".*" lacks a digest`)
+}
+
+func TestDependsImageBadDigest(t *testing.T) {
+	testBadBuildYML(t, `
+image: dummy
+depends:
+  docker-images:
+    target-dir: dl
+    list:
+      - docker.io/library/nginx:latest@sha256:invalid
+`, `unable to validate digest in ".*"`)
+}
+
+func TestDependsImageBothTargets(t *testing.T) {
+	testBadBuildYML(t, `
+image: dummy
+depends:
+  docker-images:
+    target: foo.tar
+    target-dir: dl
+`, `"depends.images.target" and "depends.images.target-dir" are mutually exclusive`)
+}
+
+func TestDependsImageBothLists(t *testing.T) {
+	testBadBuildYML(t, `
+image: dummy
+depends:
+  docker-images:
+    from-file: images.lst
+    list:
+      - one
+`, `"depends.images.list" and "depends.images.from-file" are mutually exclusive`)
+}