Merge pull request #3136 from rn/sources

Allow external directories for 'linuxkit pkg build'

docs/packages.md

@@ -37,6 +37,7 @@ A package source consists of a directory containing at least two files:
 - `image` _(string)_: *(mandatory)* The name of the image to build
 - `org` _(string)_: The hub/registry organisation to which this package belongs
 - `arches` _(list of string)_: The architectures which this package should be built for (valid entries are `GOARCH` names)
+- `extra-sources` _(list of strings)_: Additional sources for the package outside the package directory. The format is `src:dst`, where `src` can be relative to the package directory and `dst` is the destination in the build context. This is useful for sharing files, such as vendored go code, between packages.
 - `gitrepo` _(string)_: The git repository where the package source is kept.
 - `network` _(bool)_: Allow network access during the package build (default: no)
 - `disable-content-trust` _(bool)_: Disable Docker content trust for this package (default: no)

src/cmd/linuxkit/pkglib/build.go

@@ -1,12 +1,18 @@
 package pkglib
 
 import (
+	"archive/tar"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
+	log "github.com/sirupsen/logrus"
 )
 
 type buildOpts struct {
@@ -141,6 +147,8 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 		args = append(args, "--label=org.mobyproject.linuxkit.version="+version.Version)
 		args = append(args, "--label=org.mobyproject.linuxkit.revision="+version.GitCommit)
 
+		d.ctx = &buildCtx{sources: p.sources}
+
 		if err := d.build(p.Tag()+suffix, p.path, args...); err != nil {
 			return err
 		}
@@ -191,3 +199,59 @@ func (p Pkg) Build(bos ...BuildOpt) error {
 
 	return nil
 }
+
+type buildCtx struct {
+	sources []pkgSource
+}
+
+// Copy iterates over the sources, tars up the content after rewriting the paths.
+// It assumes that sources is sane, ie is well formed and the first part is an absolute path
+// and that it exists. NewFromCLI() ensures that.
+func (c *buildCtx) Copy(w io.WriteCloser) error {
+	tw := tar.NewWriter(w)
+	defer func() {
+		tw.Close()
+		w.Close()
+	}()
+
+	for _, s := range c.sources {
+		log.Debugf("Adding to build context: %s -> %s", s.src, s.dst)
+
+		f := func(p string, i os.FileInfo, err error) error {
+			if err != nil {
+				return fmt.Errorf("ctx: Walk error on %s: %v", p, err)
+			}
+
+			h, err := tar.FileInfoHeader(i, "")
+			if err != nil {
+				return fmt.Errorf("ctx: Converting FileInfo for %s: %v", p, err)
+			}
+			h.Name = path.Join(s.dst, strings.TrimPrefix(p, s.src))
+			if err := tw.WriteHeader(h); err != nil {
+				return fmt.Errorf("ctx: Writing header for %s: %v", p, err)
+			}
+
+			if !i.Mode().IsRegular() {
+				return nil
+			}
+
+			f, err := os.Open(p)
+			if err != nil {
+				return fmt.Errorf("ctx: Open %s: %v", p, err)
+			}
+			defer f.Close()
+
+			_, err = io.Copy(tw, f)
+			if err != nil {
+				return fmt.Errorf("ctx: Writing %s: %v", p, err)
+			}
+			return nil
+		}
+
+		if err := filepath.Walk(s.src, f); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

src/cmd/linuxkit/pkglib/docker.go

@@ -6,10 +6,12 @@ package pkglib
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 const dctEnableEnv = "DOCKER_CONTENT_TRUST=1"
@@ -17,6 +19,14 @@ const dctEnableEnv = "DOCKER_CONTENT_TRUST=1"
 type dockerRunner struct {
 	dct   bool
 	cache bool
+
+	// Optional build context to use
+	ctx buildContext
+}
+
+type buildContext interface {
+	// Copy copies the build context to the supplied WriterCloser
+	Copy(io.WriteCloser) error
 }
 
 func newDockerRunner(dct, cache bool) dockerRunner {
@@ -52,6 +62,8 @@ func (dr dockerRunner) command(args ...string) error {
 		dct = dctEnableEnv + " "
 	}
 
+	var eg errgroup.Group
+
 	if args[0] == "build" {
 		buildArgs := []string{}
 		for _, proxyVarName := range proxyEnvVars {
@@ -61,15 +73,30 @@ func (dr dockerRunner) command(args ...string) error {
 			}
 		}
 		cmd.Args = append(append(cmd.Args[:2], buildArgs...), cmd.Args[2:]...)
+
+		if dr.ctx != nil {
+			stdin, err := cmd.StdinPipe()
+			if err != nil {
+				return err
+			}
+			eg.Go(func() error {
+				defer stdin.Close()
+				return dr.ctx.Copy(stdin)
+			})
+
+			cmd.Args = append(cmd.Args[:len(cmd.Args)-1], "-")
+		}
 	}
 
 	log.Debugf("Executing: %s%v", dct, cmd.Args)
 
-	err := cmd.Run()
+	if err := cmd.Run(); err != nil {
-	if isExecErrNotFound(err) {
+		if isExecErrNotFound(err) {
-		return fmt.Errorf("linuxkit pkg requires docker to be installed")
+			return fmt.Errorf("linuxkit pkg requires docker to be installed")
+		}
+		return err
 	}
-	return err
+	return eg.Wait()
 }
 
 func (dr dockerRunner) pull(img string) (bool, error) {

src/cmd/linuxkit/pkglib/pkglib.go

@@ -1,11 +1,13 @@
 package pkglib
 
 import (
+	"crypto/sha1"
 	"flag"
 	"fmt"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -17,6 +19,7 @@ type pkgInfo struct {
 	Image               string            `yaml:"image"`
 	Org                 string            `yaml:"org"`
 	Arches              []string          `yaml:"arches"`
+	ExtraSources        []string          `yaml:"extra-sources"`
 	GitRepo             string            `yaml:"gitrepo"` // ??
 	Network             bool              `yaml:"network"`
 	DisableContentTrust bool              `yaml:"disable-content-trust"`
@@ -32,12 +35,19 @@ type pkgInfo struct {
 	} `yaml:"depends"`
 }
 
+// Specifies the source directory for a package and their destination in the build context.
+type pkgSource struct {
+	src string
+	dst string
+}
+
 // Pkg encapsulates information about a package's source
 type Pkg struct {
 	// These correspond to pkgInfo fields
 	image         string
 	org           string
 	arches        []string
+	sources       []pkgSource
 	gitRepo       string
 	network       bool
 	trust         bool
@@ -169,6 +179,37 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 		}
 	})
 
+	var srcHashes string
+	sources := []pkgSource{{src: pkgPath, dst: "/"}}
+
+	for _, source := range pi.ExtraSources {
+		tmp := strings.Split(source, ":")
+		if len(tmp) != 2 {
+			return Pkg{}, fmt.Errorf("Bad source format in %s", source)
+		}
+		srcPath := filepath.Clean(tmp[0]) // Should work with windows paths
+		dstPath := path.Clean(tmp[1])     // 'path' here because this should be a Unix path
+
+		if !filepath.IsAbs(srcPath) {
+			srcPath = filepath.Join(pkgPath, srcPath)
+		}
+
+		g, err := newGit(srcPath)
+		if err != nil {
+			return Pkg{}, err
+		}
+		if g == nil {
+			return Pkg{}, fmt.Errorf("Source %s not in a git repository", srcPath)
+		}
+		h, err := g.treeHash(srcPath, hashCommit)
+		if err != nil {
+			return Pkg{}, err
+		}
+
+		srcHashes += h
+		sources = append(sources, pkgSource{src: srcPath, dst: dstPath})
+	}
+
 	git, err := newGit(pkgPath)
 	if err != nil {
 		return Pkg{}, err
@@ -187,6 +228,11 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 				return Pkg{}, err
 			}
 
+			if srcHashes != "" {
+				hash += srcHashes
+				hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash)))
+			}
+
 			if dirty {
 				hash += "-dirty"
 			}
@@ -199,6 +245,7 @@ func NewFromCLI(fs *flag.FlagSet, args ...string) (Pkg, error) {
 		hash:          hash,
 		commitHash:    hashCommit,
 		arches:        pi.Arches,
+		sources:       sources,
 		gitRepo:       pi.GitRepo,
 		network:       pi.Network,
 		trust:         !pi.DisableContentTrust,