From 75149c56c297f66801e72e8c5942aa93fab2cc67 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Tue, 24 Jul 2018 14:00:19 +0100 Subject: [PATCH 1/5] cmd/pkg: Add 'extra-source' field to the pkg schema And 'sources' to the Pkg structure Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/pkglib/pkglib.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 1954a504e..ad4b07368 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -17,6 +17,7 @@ type pkgInfo struct { Image string `yaml:"image"` Org string `yaml:"org"` Arches []string `yaml:"arches"` + ExtraSources []string `yaml:"extra-sources"` GitRepo string `yaml:"gitrepo"` // ?? Network bool `yaml:"network"` DisableContentTrust bool `yaml:"disable-content-trust"` @@ -32,12 +33,19 @@ type pkgInfo struct { } `yaml:"depends"` } +// Specifies the source directory for a package and their destination in the build context. +type pkgSource struct { + src string + dst string +} + // Pkg encapsulates information about a package's source type Pkg struct { // These correspond to pkgInfo fields image string org string arches []string + sources []pkgSource gitRepo string network bool trust bool @@ -199,6 +207,7 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { hash: hash, commitHash: hashCommit, arches: pi.Arches, + sources: pi.Sources, gitRepo: pi.GitRepo, network: pi.Network, trust: !pi.DisableContentTrust, From ba4d1c79b07d6092202590451d27292dc194767c Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Tue, 24 Jul 2018 15:25:15 +0100 Subject: [PATCH 2/5] cmd/pkg: Extract 'extra-sources' and adjust hash calculation If the build.yml specifies 'extra-sources', ie sources outside the package directory, calculate the hash based on the tree hash of all source directories and the package directory. Note, this requires the source directories to be under git revision control. Also clean up the src and dst of the path and stash the result in the Pkg structure. Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/pkglib/pkglib.go | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index ad4b07368..c5c1108a5 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -1,11 +1,13 @@ package pkglib import ( + "crypto/sha1" "flag" "fmt" "gopkg.in/yaml.v2" "io/ioutil" "os" + "path" "path/filepath" "strings" @@ -177,6 +179,37 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { } }) + var srcHashes string + sources := []pkgSource{{src: pkgPath, dst: "/"}} + + for _, source := range pi.ExtraSources { + tmp := strings.Split(source, ":") + if len(tmp) != 2 { + return Pkg{}, fmt.Errorf("Bad source format in %s", source) + } + srcPath := filepath.Clean(tmp[0]) // Should work with windows paths + dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path + + if !filepath.IsAbs(srcPath) { + srcPath = filepath.Join(pkgPath, srcPath) + } + + g, err := newGit(srcPath) + if err != nil { + return Pkg{}, err + } + if g == nil { + return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath) + } + h, err := g.treeHash(srcPath, hashCommit) + if err != nil { + return Pkg{}, err + } + + srcHashes += h + sources = append(sources, pkgSource{src: srcPath, dst: dstPath}) + } + git, err := newGit(pkgPath) if err != nil { return Pkg{}, err @@ -195,6 +228,11 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { return Pkg{}, err } + if srcHashes != "" { + hash += srcHashes + hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash))) + } + if dirty { hash += "-dirty" } @@ -207,7 +245,7 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { hash: hash, commitHash: hashCommit, arches: pi.Arches, - sources: pi.Sources, + sources: sources, gitRepo: pi.GitRepo, network: pi.Network, trust: !pi.DisableContentTrust, From b03288f5b4345ec03dfd17ac4304830eef36af8d Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Wed, 25 Jul 2018 13:24:35 +0100 Subject: [PATCH 3/5] cmd/pkg: Add the ability to pass a build context to docker This commit adds the ability to add a build context to docker for the package build. The build context is passed on stdin to the docker process. Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/pkglib/docker.go | 35 +++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/cmd/linuxkit/pkglib/docker.go b/src/cmd/linuxkit/pkglib/docker.go index afd1e96db..b499fd7ec 100644 --- a/src/cmd/linuxkit/pkglib/docker.go +++ b/src/cmd/linuxkit/pkglib/docker.go @@ -6,10 +6,12 @@ package pkglib import ( "fmt" + "io" "os" "os/exec" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) const dctEnableEnv = "DOCKER_CONTENT_TRUST=1" @@ -17,6 +19,14 @@ const dctEnableEnv = "DOCKER_CONTENT_TRUST=1" type dockerRunner struct { dct bool cache bool + + // Optional build context to use + ctx buildContext +} + +type buildContext interface { + // Copy copies the build context to the supplied WriterCloser + Copy(io.WriteCloser) error } func newDockerRunner(dct, cache bool) dockerRunner { @@ -52,6 +62,8 @@ func (dr dockerRunner) command(args ...string) error { dct = dctEnableEnv + " " } + var eg errgroup.Group + if args[0] == "build" { buildArgs := []string{} for _, proxyVarName := range proxyEnvVars { @@ -61,15 +73,30 @@ func (dr dockerRunner) command(args ...string) error { } } cmd.Args = append(append(cmd.Args[:2], buildArgs...), cmd.Args[2:]...) + + if dr.ctx != nil { + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + eg.Go(func() error { + defer stdin.Close() + return dr.ctx.Copy(stdin) + }) + + cmd.Args = append(cmd.Args[:len(cmd.Args)-1], "-") + } } log.Debugf("Executing: %s%v", dct, cmd.Args) - err := cmd.Run() - if isExecErrNotFound(err) { - return fmt.Errorf("linuxkit pkg requires docker to be installed") + if err := cmd.Run(); err != nil { + if isExecErrNotFound(err) { + return fmt.Errorf("linuxkit pkg requires docker to be installed") + } + return err } - return err + return eg.Wait() } func (dr dockerRunner) pull(img string) (bool, error) { From bebde3a2ea9f49c7da5ed8a7e7123c51ecf87d1e Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Wed, 25 Jul 2018 16:17:05 +0100 Subject: [PATCH 4/5] cmd/pkg: Build a build context from 'sources' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the final piece. If 'sources' are defined, tar up the sources and rewrite them accordingly. Pass it as build build context to 'docker'. This allows building from something like this: ├── etc │   ├── foo └── foo ├── Dockerfile ├── build.yml └── main.go With 'build.yml': image: foo extra-sources: - ../etc:etc Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/pkglib/build.go | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index fee17a572..9a8567efc 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -1,12 +1,18 @@ package pkglib import ( + "archive/tar" "encoding/json" "fmt" + "io" "os" + "path" + "path/filepath" "runtime" + "strings" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" + log "github.com/sirupsen/logrus" ) type buildOpts struct { @@ -141,6 +147,8 @@ func (p Pkg) Build(bos ...BuildOpt) error { args = append(args, "--label=org.mobyproject.linuxkit.version="+version.Version) args = append(args, "--label=org.mobyproject.linuxkit.revision="+version.GitCommit) + d.ctx = &buildCtx{sources: p.sources} + if err := d.build(p.Tag()+suffix, p.path, args...); err != nil { return err } @@ -191,3 +199,59 @@ func (p Pkg) Build(bos ...BuildOpt) error { return nil } + +type buildCtx struct { + sources []pkgSource +} + +// Copy iterates over the sources, tars up the content after rewriting the paths. +// It assumes that sources is sane, ie is well formed and the first part is an absolute path +// and that it exists. NewFromCLI() ensures that. +func (c *buildCtx) Copy(w io.WriteCloser) error { + tw := tar.NewWriter(w) + defer func() { + tw.Close() + w.Close() + }() + + for _, s := range c.sources { + log.Debugf("Adding to build context: %s -> %s", s.src, s.dst) + + f := func(p string, i os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("ctx: Walk error on %s: %v", p, err) + } + + h, err := tar.FileInfoHeader(i, "") + if err != nil { + return fmt.Errorf("ctx: Converting FileInfo for %s: %v", p, err) + } + h.Name = path.Join(s.dst, strings.TrimPrefix(p, s.src)) + if err := tw.WriteHeader(h); err != nil { + return fmt.Errorf("ctx: Writing header for %s: %v", p, err) + } + + if !i.Mode().IsRegular() { + return nil + } + + f, err := os.Open(p) + if err != nil { + return fmt.Errorf("ctx: Open %s: %v", p, err) + } + defer f.Close() + + _, err = io.Copy(tw, f) + if err != nil { + return fmt.Errorf("ctx: Writing %s: %v", p, err) + } + return nil + } + + if err := filepath.Walk(s.src, f); err != nil { + return err + } + } + + return nil +} From 29dd9f2004f56ed151e65cf23d761e3b6db4e48e Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Wed, 25 Jul 2018 16:55:14 +0100 Subject: [PATCH 5/5] doc: Add 'sources' documentation Signed-off-by: Rolf Neugebauer --- docs/packages.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/packages.md b/docs/packages.md index 87bc9911b..a4330ebbd 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -37,6 +37,7 @@ A package source consists of a directory containing at least two files: - `image` _(string)_: *(mandatory)* The name of the image to build - `org` _(string)_: The hub/registry organisation to which this package belongs - `arches` _(list of string)_: The architectures which this package should be built for (valid entries are `GOARCH` names) +- `extra-sources` _(list of strings)_: Additional sources for the package outside the package directory. The format is `src:dst`, where `src` can be relative to the package directory and `dst` is the destination in the build context. This is useful for sharing files, such as vendored go code, between packages. - `gitrepo` _(string)_: The git repository where the package source is kept. - `network` _(bool)_: Allow network access during the package build (default: no) - `disable-content-trust` _(bool)_: Disable Docker content trust for this package (default: no)