diff --git a/docs/packages.md b/docs/packages.md index 4a0a024b2..67977e99c 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -30,6 +30,10 @@ A package source consists of a directory containing at least two files: - `disable-content-trust` _(bool)_: Disable Docker content trust for this package (default: no) - `disable-cache` _(bool)_: Disable build cache for this package (default: no) - `config`: _(struct `github.com/moby/tool/src/moby.ImageConfig`)_: Image configuration, marshalled to JSON and added as `org.mobyproject.config` label on image (default: no label) +- `depends`: Contains information on prerequisites which must be satisfied in order to build the package. Has subfields: + - `docker-images`: Docker images to be made available (as `tar` files via `docker image save`) within the package build context. Contains the following nested fields: + - `from-file` and `list`: _(string and string list respectively)_. Mutually exclusive fields specifying the list of images to include. Each image must include a valid digest (`sha256:...`) in order to maintain determinism. If `from-file` is used then it is a path relative to (and within) the package directory with one image per line (lines with `#` in column 0 and blank lines are ignore). If `list` is used then each entry is an image. + - `target` and `target-dir`: _(string)_ Mutually exclusive fields specifying the target location, if `target` is used then it is a path relative to (and within) the package dir which names a `tar` file into which all of the listed images will be saved. If `target-dir` then it is a path relative to (and within) the package directory which names a directory into which each image will be saved (as `«image name»@«digest».tar`). **NB**: The path referenced by `target-dir` will be _removed_ prior to populating (to avoid issues with stale files). ## Building packages diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index f399409e5..82dcacc03 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -110,6 +110,10 @@ func (p Pkg) Build(bos ...BuildOpt) error { if !bo.skipBuild { var args []string + if err := p.dockerDepends.Do(d); err != nil { + return err + } + if p.git != nil && p.gitRepo != "" { args = append(args, "--label", "org.opencontainers.image.source="+p.gitRepo) } diff --git a/src/cmd/linuxkit/pkglib/depends.go b/src/cmd/linuxkit/pkglib/depends.go new file mode 100644 index 000000000..f910bf535 --- /dev/null +++ b/src/cmd/linuxkit/pkglib/depends.go @@ -0,0 +1,133 @@ +package pkglib + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + + "github.com/containerd/containerd/reference" +) + +type dockerDepends struct { + images []reference.Spec + path string + dir bool +} + +func newDockerDepends(pkgPath string, pi *pkgInfo) (dockerDepends, error) { + var err error + + if (pi.Depends.DockerImages.TargetDir != "") && (pi.Depends.DockerImages.Target != "") { + return dockerDepends{}, fmt.Errorf("\"depends.images.target\" and \"depends.images.target-dir\" are mutually exclusive") + } + if (pi.Depends.DockerImages.FromFile != "") && (len(pi.Depends.DockerImages.List) > 0) { + return dockerDepends{}, fmt.Errorf("\"depends.images.list\" and \"depends.images.from-file\" are mutually exclusive") + } + + if pi.Depends.DockerImages.Target, err = makeAbsSubpath("depends.image.target", pkgPath, pi.Depends.DockerImages.Target); err != nil { + return dockerDepends{}, err + } + if pi.Depends.DockerImages.TargetDir, err = makeAbsSubpath("depends.image.target-dir", pkgPath, pi.Depends.DockerImages.TargetDir); err != nil { + return dockerDepends{}, err + } + if pi.Depends.DockerImages.FromFile != "" { + p, err := makeAbsSubpath("depends.image.from-file", pkgPath, pi.Depends.DockerImages.FromFile) + if err != nil { + return dockerDepends{}, err + } + f, err := os.Open(p) + if err != nil { + return dockerDepends{}, err + } + defer f.Close() + + s := bufio.NewScanner(f) + for s.Scan() { + t := s.Text() + if len(t) > 0 && t[0] != '#' { + pi.Depends.DockerImages.List = append(pi.Depends.DockerImages.List, s.Text()) + } + } + + if err := s.Err(); err != nil { + return dockerDepends{}, err + } + } + + var specs []reference.Spec + for _, i := range pi.Depends.DockerImages.List { + s, err := reference.Parse(i) + if err != nil { + return dockerDepends{}, err + } + dgst := s.Digest() + if dgst == "" { + return dockerDepends{}, fmt.Errorf("image %q lacks a digest", i) + } + if err := dgst.Validate(); err != nil { + return dockerDepends{}, fmt.Errorf("unable to validate digest in %q: %v", i, err) + } + specs = append(specs, s) + } + + var dir bool + path := pi.Depends.DockerImages.Target + if pi.Depends.DockerImages.TargetDir != "" { + path = pi.Depends.DockerImages.TargetDir + dir = true + } + return dockerDepends{ + images: specs, + path: path, + dir: dir, + }, nil +} + +// Do ensures that any dependencies the package has declared are met. +func (dd dockerDepends) Do(d dockerRunner) error { + if len(dd.images) == 0 { + return nil + } + + if dd.dir { + dir := dd.path + + // Delete and recreate so it is empty + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("failed to remove %q: %v", dir, err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create %q: %v", dir, err) + } + } + + var refs []string + + for _, s := range dd.images { + if ok, err := d.pull(s.String()); !ok || err != nil { + if err != nil { + return err + } + return fmt.Errorf("failed to pull %q", s.String()) + } + + refs = append(refs, s.Locator) + if dd.dir { + bn := filepath.Base(s.Locator) + "@" + s.Digest().String() + path := filepath.Join(dd.path, bn+".tar") + fmt.Printf("Adding %q as dependency\n", bn) + if err := d.save(path, s.String()); err != nil { + return err + } + } + } + + if !dd.dir { + if err := d.save(dd.path, refs...); err != nil { + return err + } + } + + return nil +} diff --git a/src/cmd/linuxkit/pkglib/docker.go b/src/cmd/linuxkit/pkglib/docker.go index 154bdecd3..2d8e5a901 100644 --- a/src/cmd/linuxkit/pkglib/docker.go +++ b/src/cmd/linuxkit/pkglib/docker.go @@ -103,3 +103,8 @@ func (dr dockerRunner) build(tag, pkg string, opts ...string) error { args = append(args, "-t", tag, pkg) return dr.command(args...) } + +func (dr dockerRunner) save(tgt string, refs ...string) error { + args := append([]string{"image", "save", "-o", tgt}, refs...) + return dr.command(args...) +} diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 4be87a76e..15fdddcb7 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -12,7 +12,7 @@ import ( "github.com/moby/tool/src/moby" ) -// Containers fields settable in the build.yml +// Contains fields settable in the build.yml type pkgInfo struct { Image string `yaml:"image"` Org string `yaml:"org"` @@ -22,19 +22,28 @@ type pkgInfo struct { DisableContentTrust bool `yaml:"disable-content-trust"` DisableCache bool `yaml:"disable-cache"` Config *moby.ImageConfig `yaml:"config"` + Depends struct { + DockerImages struct { + TargetDir string `yaml:"target-dir"` + Target string `yaml:"target"` + FromFile string `yaml:"from-file"` + List []string `yaml:"list"` + } `yaml:"docker-images"` + } `yaml:"depends"` } // Pkg encapsulates information about a package's source type Pkg struct { // These correspond to pkgInfo fields - image string - org string - arches []string - gitRepo string - network bool - trust bool - cache bool - config *moby.ImageConfig + image string + org string + arches []string + gitRepo string + network bool + trust bool + cache bool + config *moby.ImageConfig + dockerDepends dockerDepends // Internal state path string @@ -123,6 +132,11 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { return Pkg{}, fmt.Errorf("Image field is required") } + dockerDepends, err := newDockerDepends(pkgPath, &pi) + if err != nil { + return Pkg{}, err + } + if devMode { // If --org is also used then this will be overwritten // by argOrg when we iterate over the provided options @@ -180,19 +194,20 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { } return Pkg{ - image: pi.Image, - org: pi.Org, - hash: hash, - commitHash: hashCommit, - arches: pi.Arches, - gitRepo: pi.GitRepo, - network: pi.Network, - trust: !pi.DisableContentTrust, - cache: !pi.DisableCache, - config: pi.Config, - dirty: dirty, - path: pkgPath, - git: git, + image: pi.Image, + org: pi.Org, + hash: hash, + commitHash: hashCommit, + arches: pi.Arches, + gitRepo: pi.GitRepo, + network: pi.Network, + trust: !pi.DisableContentTrust, + cache: !pi.DisableCache, + config: pi.Config, + dockerDepends: dockerDepends, + dirty: dirty, + path: pkgPath, + git: git, }, nil } @@ -242,3 +257,29 @@ func (p Pkg) cleanForBuild() error { } return nil } + +// Expands path from relative to abs against base, ensuring the result is within base, but is not base itself. Field is the fieldname, to be used for constructing the error. +func makeAbsSubpath(field, base, path string) (string, error) { + if path == "" { + return "", nil + } + + if filepath.IsAbs(path) { + return "", fmt.Errorf("%s must be relative to package directory", field) + } + + p, err := filepath.Abs(filepath.Join(base, path)) + if err != nil { + return "", err + } + + if p == base { + return "", fmt.Errorf("%s must not be exactly the package directory", field) + } + + if !filepath.HasPrefix(p, base) { + return "", fmt.Errorf("%s must be within package directory", field) + } + + return p, nil +} diff --git a/src/cmd/linuxkit/pkglib/pkglib_test.go b/src/cmd/linuxkit/pkglib/pkglib_test.go index 45ef07709..cc432f97b 100644 --- a/src/cmd/linuxkit/pkglib/pkglib_test.go +++ b/src/cmd/linuxkit/pkglib/pkglib_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "testing" "github.com/stretchr/testify/assert" @@ -85,3 +86,63 @@ func TestCache(t *testing.T) { func TestContentTrust(t *testing.T) { testBool(t, "disable-content-trust", true, "-enable-content-trust", "-disable-content-trust", func(p Pkg) bool { return p.trust }) } + +func testBadBuildYML(t *testing.T, build, expect string) { + cwd, err := os.Getwd() + require.NoError(t, err) + + tmpDir := filepath.Join(cwd, t.Name()) + err = os.Mkdir(tmpDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + pkgDir := dummyPackage(t, tmpDir, build) + flags := flag.NewFlagSet(t.Name(), flag.ExitOnError) + args := []string{"-hash-path=" + cwd, pkgDir} + _, err = NewFromCLI(flags, args...) + require.Error(t, err) + assert.Regexp(t, regexp.MustCompile(expect), err.Error()) +} + +func TestDependsImageNoDigest(t *testing.T) { + testBadBuildYML(t, ` +image: dummy +depends: + docker-images: + target-dir: dl + list: + - docker.io/library/nginx:latest +`, `image ".*" lacks a digest`) +} + +func TestDependsImageBadDigest(t *testing.T) { + testBadBuildYML(t, ` +image: dummy +depends: + docker-images: + target-dir: dl + list: + - docker.io/library/nginx:latest@sha256:invalid +`, `unable to validate digest in ".*"`) +} + +func TestDependsImageBothTargets(t *testing.T) { + testBadBuildYML(t, ` +image: dummy +depends: + docker-images: + target: foo.tar + target-dir: dl +`, `"depends.images.target" and "depends.images.target-dir" are mutually exclusive`) +} + +func TestDependsImageBothLists(t *testing.T) { + testBadBuildYML(t, ` +image: dummy +depends: + docker-images: + from-file: images.lst + list: + - one +`, `"depends.images.list" and "depends.images.from-file" are mutually exclusive`) +}