diff --git a/docs/packages.md b/docs/packages.md index 87bc9911b..a4330ebbd 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -37,6 +37,7 @@ A package source consists of a directory containing at least two files: - `image` _(string)_: *(mandatory)* The name of the image to build - `org` _(string)_: The hub/registry organisation to which this package belongs - `arches` _(list of string)_: The architectures which this package should be built for (valid entries are `GOARCH` names) +- `extra-sources` _(list of strings)_: Additional sources for the package outside the package directory. The format is `src:dst`, where `src` can be relative to the package directory and `dst` is the destination in the build context. This is useful for sharing files, such as vendored go code, between packages. - `gitrepo` _(string)_: The git repository where the package source is kept. - `network` _(bool)_: Allow network access during the package build (default: no) - `disable-content-trust` _(bool)_: Disable Docker content trust for this package (default: no) diff --git a/src/cmd/linuxkit/pkglib/build.go b/src/cmd/linuxkit/pkglib/build.go index fee17a572..9a8567efc 100644 --- a/src/cmd/linuxkit/pkglib/build.go +++ b/src/cmd/linuxkit/pkglib/build.go @@ -1,12 +1,18 @@ package pkglib import ( + "archive/tar" "encoding/json" "fmt" + "io" "os" + "path" + "path/filepath" "runtime" + "strings" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/version" + log "github.com/sirupsen/logrus" ) type buildOpts struct { @@ -141,6 +147,8 @@ func (p Pkg) Build(bos ...BuildOpt) error { args = append(args, "--label=org.mobyproject.linuxkit.version="+version.Version) args = append(args, "--label=org.mobyproject.linuxkit.revision="+version.GitCommit) + d.ctx = &buildCtx{sources: p.sources} + if err := d.build(p.Tag()+suffix, p.path, args...); err != nil { return err } @@ -191,3 +199,59 @@ func (p Pkg) Build(bos ...BuildOpt) error { return nil } + +type buildCtx struct { + sources []pkgSource +} + +// Copy iterates over the sources, tars up the content after rewriting the paths. +// It assumes that sources is sane, ie is well formed and the first part is an absolute path +// and that it exists. NewFromCLI() ensures that. +func (c *buildCtx) Copy(w io.WriteCloser) error { + tw := tar.NewWriter(w) + defer func() { + tw.Close() + w.Close() + }() + + for _, s := range c.sources { + log.Debugf("Adding to build context: %s -> %s", s.src, s.dst) + + f := func(p string, i os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("ctx: Walk error on %s: %v", p, err) + } + + h, err := tar.FileInfoHeader(i, "") + if err != nil { + return fmt.Errorf("ctx: Converting FileInfo for %s: %v", p, err) + } + h.Name = path.Join(s.dst, strings.TrimPrefix(p, s.src)) + if err := tw.WriteHeader(h); err != nil { + return fmt.Errorf("ctx: Writing header for %s: %v", p, err) + } + + if !i.Mode().IsRegular() { + return nil + } + + f, err := os.Open(p) + if err != nil { + return fmt.Errorf("ctx: Open %s: %v", p, err) + } + defer f.Close() + + _, err = io.Copy(tw, f) + if err != nil { + return fmt.Errorf("ctx: Writing %s: %v", p, err) + } + return nil + } + + if err := filepath.Walk(s.src, f); err != nil { + return err + } + } + + return nil +} diff --git a/src/cmd/linuxkit/pkglib/docker.go b/src/cmd/linuxkit/pkglib/docker.go index afd1e96db..b499fd7ec 100644 --- a/src/cmd/linuxkit/pkglib/docker.go +++ b/src/cmd/linuxkit/pkglib/docker.go @@ -6,10 +6,12 @@ package pkglib import ( "fmt" + "io" "os" "os/exec" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) const dctEnableEnv = "DOCKER_CONTENT_TRUST=1" @@ -17,6 +19,14 @@ const dctEnableEnv = "DOCKER_CONTENT_TRUST=1" type dockerRunner struct { dct bool cache bool + + // Optional build context to use + ctx buildContext +} + +type buildContext interface { + // Copy copies the build context to the supplied WriterCloser + Copy(io.WriteCloser) error } func newDockerRunner(dct, cache bool) dockerRunner { @@ -52,6 +62,8 @@ func (dr dockerRunner) command(args ...string) error { dct = dctEnableEnv + " " } + var eg errgroup.Group + if args[0] == "build" { buildArgs := []string{} for _, proxyVarName := range proxyEnvVars { @@ -61,15 +73,30 @@ func (dr dockerRunner) command(args ...string) error { } } cmd.Args = append(append(cmd.Args[:2], buildArgs...), cmd.Args[2:]...) + + if dr.ctx != nil { + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + eg.Go(func() error { + defer stdin.Close() + return dr.ctx.Copy(stdin) + }) + + cmd.Args = append(cmd.Args[:len(cmd.Args)-1], "-") + } } log.Debugf("Executing: %s%v", dct, cmd.Args) - err := cmd.Run() - if isExecErrNotFound(err) { - return fmt.Errorf("linuxkit pkg requires docker to be installed") + if err := cmd.Run(); err != nil { + if isExecErrNotFound(err) { + return fmt.Errorf("linuxkit pkg requires docker to be installed") + } + return err } - return err + return eg.Wait() } func (dr dockerRunner) pull(img string) (bool, error) { diff --git a/src/cmd/linuxkit/pkglib/pkglib.go b/src/cmd/linuxkit/pkglib/pkglib.go index 1954a504e..c5c1108a5 100644 --- a/src/cmd/linuxkit/pkglib/pkglib.go +++ b/src/cmd/linuxkit/pkglib/pkglib.go @@ -1,11 +1,13 @@ package pkglib import ( + "crypto/sha1" "flag" "fmt" "gopkg.in/yaml.v2" "io/ioutil" "os" + "path" "path/filepath" "strings" @@ -17,6 +19,7 @@ type pkgInfo struct { Image string `yaml:"image"` Org string `yaml:"org"` Arches []string `yaml:"arches"` + ExtraSources []string `yaml:"extra-sources"` GitRepo string `yaml:"gitrepo"` // ?? Network bool `yaml:"network"` DisableContentTrust bool `yaml:"disable-content-trust"` @@ -32,12 +35,19 @@ type pkgInfo struct { } `yaml:"depends"` } +// Specifies the source directory for a package and their destination in the build context. +type pkgSource struct { + src string + dst string +} + // Pkg encapsulates information about a package's source type Pkg struct { // These correspond to pkgInfo fields image string org string arches []string + sources []pkgSource gitRepo string network bool trust bool @@ -169,6 +179,37 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { } }) + var srcHashes string + sources := []pkgSource{{src: pkgPath, dst: "/"}} + + for _, source := range pi.ExtraSources { + tmp := strings.Split(source, ":") + if len(tmp) != 2 { + return Pkg{}, fmt.Errorf("Bad source format in %s", source) + } + srcPath := filepath.Clean(tmp[0]) // Should work with windows paths + dstPath := path.Clean(tmp[1]) // 'path' here because this should be a Unix path + + if !filepath.IsAbs(srcPath) { + srcPath = filepath.Join(pkgPath, srcPath) + } + + g, err := newGit(srcPath) + if err != nil { + return Pkg{}, err + } + if g == nil { + return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath) + } + h, err := g.treeHash(srcPath, hashCommit) + if err != nil { + return Pkg{}, err + } + + srcHashes += h + sources = append(sources, pkgSource{src: srcPath, dst: dstPath}) + } + git, err := newGit(pkgPath) if err != nil { return Pkg{}, err @@ -187,6 +228,11 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { return Pkg{}, err } + if srcHashes != "" { + hash += srcHashes + hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash))) + } + if dirty { hash += "-dirty" } @@ -199,6 +245,7 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) { hash: hash, commitHash: hashCommit, arches: pi.Arches, + sources: sources, gitRepo: pi.GitRepo, network: pi.Network, trust: !pi.DisableContentTrust,