From e7ebabdb059fe4041670fbd5b062e3943bba4155 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Wed, 21 Jun 2017 16:19:31 -0700 Subject: [PATCH] Split out into a small stub command line and a library - this is pretty much the smallest change to split this out and it exposes a few things that can be improved later - no change to logging yet Signed-off-by: Justin Cormack --- Makefile | 2 +- cmd/moby/build.go | 500 +++---------------------------- cmd/moby/main.go | 15 +- src/moby/build.go | 467 +++++++++++++++++++++++++++++ {cmd => src}/moby/config.go | 22 +- {cmd => src}/moby/config_test.go | 6 +- {cmd => src}/moby/docker.go | 2 +- {cmd => src}/moby/image.go | 2 +- {cmd => src}/moby/linuxkit.go | 4 +- {cmd => src}/moby/output.go | 10 +- {cmd => src}/moby/schema.go | 2 +- {cmd => src}/moby/trust.go | 2 +- {cmd => src}/moby/trust_test.go | 2 +- src/moby/util.go | 6 + src/moby/util_unix.go | 11 + src/moby/util_windows.go | 9 + 16 files changed, 564 insertions(+), 498 deletions(-) create mode 100644 src/moby/build.go rename {cmd => src}/moby/config.go (97%) rename {cmd => src}/moby/config_test.go (94%) rename {cmd => src}/moby/docker.go (99%) rename {cmd => src}/moby/image.go (99%) rename {cmd => src}/moby/linuxkit.go (98%) rename {cmd => src}/moby/output.go (96%) rename {cmd => src}/moby/schema.go (99%) rename {cmd => src}/moby/trust.go (99%) rename {cmd => src}/moby/trust_test.go (99%) create mode 100644 src/moby/util.go create mode 100644 src/moby/util_unix.go create mode 100644 src/moby/util_windows.go diff --git a/Makefile b/Makefile index de76352be..50c65fb3a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ lint: # govet @test -z "$$(go tool vet -printf=false . 2>&1 | grep -v vendor/ | tee /dev/stderr)" # go test - @go test github.com/moby/tool/cmd/moby + @go test github.com/moby/tool/src/moby test: moby ./moby build -output tar test/test.yml diff --git a/cmd/moby/build.go b/cmd/moby/build.go index 99f168602..4daa0504b 100644 --- a/cmd/moby/build.go +++ b/cmd/moby/build.go @@ -1,23 +1,20 @@ package main import ( - "archive/tar" "bytes" - "errors" "flag" "fmt" "io" "io/ioutil" "net/http" "os" - "path" "path/filepath" "runtime" - "sort" "strconv" "strings" log "github.com/Sirupsen/logrus" + "github.com/moby/tool/src/moby" ) const defaultNameForStdin = "moby" @@ -36,52 +33,32 @@ func (o *outputList) Set(value string) error { return nil } -var streamable = map[string]bool{ - "docker": true, - "tar": true, -} - -type addFun func(*tar.Writer) error - -const dockerfile = ` -FROM scratch - -COPY . ./ -RUN rm -f Dockerfile - -ENTRYPOINT ["/sbin/tini", "--", "/bin/rc.init"] -` - -var additions = map[string]addFun{ - "docker": func(tw *tar.Writer) error { - log.Infof(" Adding Dockerfile") - hdr := &tar.Header{ - Name: "Dockerfile", - Mode: 0644, - Size: int64(len(dockerfile)), +// Parse a string which is either a number in MB, or a number with +// either M (for Megabytes) or G (for GigaBytes) as a suffix and +// returns the number in MB. Return 0 if string is empty. +func getDiskSizeMB(s string) (int, error) { + if s == "" { + return 0, nil + } + sz := len(s) + if strings.HasSuffix(s, "G") { + i, err := strconv.Atoi(s[:sz-1]) + if err != nil { + return 0, err } - if err := tw.WriteHeader(hdr); err != nil { - return err - } - if _, err := tw.Write([]byte(dockerfile)); err != nil { - return err - } - return nil - }, + return i * 1024, nil + } + if strings.HasSuffix(s, "M") { + s = s[:sz-1] + } + return strconv.Atoi(s) } // Process the build arguments and execute build func build(args []string) { var buildOut outputList - outputTypes := []string{} - for k := range streamable { - outputTypes = append(outputTypes, k) - } - for k := range outFuns { - outputTypes = append(outputTypes, k) - } - sort.Strings(outputTypes) + outputTypes := moby.OutputTypes() buildCmd := flag.NewFlagSet("build", flag.ExitOnError) buildCmd.Usage = func() { @@ -135,13 +112,13 @@ func build(args []string) { if len(buildOut) > 1 { for _, o := range buildOut { - if streamable[o] { + if moby.Streamable(o) { log.Fatalf("Output type %s must be the only output specified", o) } } } - if len(buildOut) == 1 && streamable[buildOut[0]] { + if len(buildOut) == 1 && moby.Streamable(buildOut[0]) { if *buildOutputFile == "" { *buildOutputFile = filepath.Join(*buildDir, name+"."+buildOut[0]) // stop the errors in the validation below @@ -150,7 +127,7 @@ func build(args []string) { } } else { - err := validateOutputs(buildOut) + err := moby.ValidateOutputs(buildOut) if err != nil { log.Errorf("Error parsing outputs: %v", err) buildCmd.Usage() @@ -159,7 +136,6 @@ func build(args []string) { } var outputFile *os.File - var addition addFun if *buildOutputFile != "" { if len(buildOut) > 1 { log.Fatal("The -output option can only be specified when generating a single output format") @@ -170,7 +146,7 @@ func build(args []string) { if *buildDir != "" { log.Fatal("The -output option cannot be specified with -dir") } - if !streamable[buildOut[0]] { + if !moby.Streamable(buildOut[0]) { log.Fatalf("The -output option cannot be specified for build type %s as it cannot be streamed", buildOut[0]) } if *buildOutputFile == "-" { @@ -183,7 +159,6 @@ func build(args []string) { } defer outputFile.Close() } - addition = additions[buildOut[0]] } size, err := getDiskSizeMB(*buildSize) @@ -191,7 +166,7 @@ func build(args []string) { log.Fatalf("Unable to parse disk size: %v", err) } - var moby Moby + var m moby.Moby for _, arg := range remArgs { var config []byte if conf := arg; conf == "-" { @@ -220,16 +195,16 @@ func build(args []string) { } } - m, err := NewConfig(config) + c, err := moby.NewConfig(config) if err != nil { log.Fatalf("Invalid config: %v", err) } - moby = AppendConfig(moby, m) + m = moby.AppendConfig(m, c) } if *buildDisableTrust { log.Debugf("Disabling content trust checks for this build") - moby.Trust = TrustConfig{} + m.Trust = moby.TrustConfig{} } var buf *bytes.Buffer @@ -240,7 +215,13 @@ func build(args []string) { buf = new(bytes.Buffer) w = buf } - err = buildInternal(moby, w, *buildPull, addition) + // this is a weird interface, but currently only streamable types can have additional files + // need to split up the base tarball outputs from the secondary stages + var tp string + if moby.Streamable(buildOut[0]) { + tp = buildOut[0] + } + err = moby.Build(m, w, *buildPull, tp) if err != nil { log.Fatalf("%v", err) } @@ -248,418 +229,9 @@ func build(args []string) { if outputFile == nil { image := buf.Bytes() log.Infof("Create outputs:") - err = outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit) + err = moby.Outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit) if err != nil { log.Fatalf("Error writing outputs: %v", err) } } } - -// Parse a string which is either a number in MB, or a number with -// either M (for Megabytes) or G (for GigaBytes) as a suffix and -// returns the number in MB. Return 0 if string is empty. -func getDiskSizeMB(s string) (int, error) { - if s == "" { - return 0, nil - } - sz := len(s) - if strings.HasSuffix(s, "G") { - i, err := strconv.Atoi(s[:sz-1]) - if err != nil { - return 0, err - } - return i * 1024, nil - } - if strings.HasSuffix(s, "M") { - s = s[:sz-1] - } - return strconv.Atoi(s) -} - -func enforceContentTrust(fullImageName string, config *TrustConfig) bool { - for _, img := range config.Image { - // First check for an exact name match - if img == fullImageName { - return true - } - // Also check for an image name only match - // by removing a possible tag (with possibly added digest): - imgAndTag := strings.Split(fullImageName, ":") - if len(imgAndTag) >= 2 && img == imgAndTag[0] { - return true - } - // and by removing a possible digest: - imgAndDigest := strings.Split(fullImageName, "@sha256:") - if len(imgAndDigest) >= 2 && img == imgAndDigest[0] { - return true - } - } - - for _, org := range config.Org { - var imgOrg string - splitName := strings.Split(fullImageName, "/") - switch len(splitName) { - case 0: - // if the image is empty, return false - return false - case 1: - // for single names like nginx, use library - imgOrg = "library" - case 2: - // for names that assume docker hub, like linxukit/alpine, take the first split - imgOrg = splitName[0] - default: - // for names that include the registry, the second piece is the org, ex: docker.io/library/alpine - imgOrg = splitName[1] - } - if imgOrg == org { - return true - } - } - return false -} - -// Perform the actual build process -func buildInternal(m Moby, w io.Writer, pull bool, addition addFun) error { - iw := tar.NewWriter(w) - - if m.Kernel.Image != "" { - // get kernel and initrd tarball from container - log.Infof("Extract kernel image: %s", m.Kernel.Image) - kf := newKernelFilter(iw, m.Kernel.Cmdline) - err := ImageTar(m.Kernel.Image, "", kf, enforceContentTrust(m.Kernel.Image, &m.Trust), pull) - if err != nil { - return fmt.Errorf("Failed to extract kernel image and tarball: %v", err) - } - err = kf.Close() - if err != nil { - return fmt.Errorf("Close error: %v", err) - } - } - - // convert init images to tarballs - if len(m.Init) != 0 { - log.Infof("Add init containers:") - } - for _, ii := range m.Init { - log.Infof("Process init image: %s", ii) - err := ImageTar(ii, "", iw, enforceContentTrust(ii, &m.Trust), pull) - if err != nil { - return fmt.Errorf("Failed to build init tarball from %s: %v", ii, err) - } - } - - if len(m.Onboot) != 0 { - log.Infof("Add onboot containers:") - } - for i, image := range m.Onboot { - log.Infof(" Create OCI config for %s", image.Image) - useTrust := enforceContentTrust(image.Image, &m.Trust) - config, err := ConfigToOCI(image, useTrust) - if err != nil { - return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err) - } - so := fmt.Sprintf("%03d", i) - path := "containers/onboot/" + so + "-" + image.Name - err = ImageBundle(path, image.Image, config, iw, useTrust, pull) - if err != nil { - return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err) - } - } - - if len(m.Services) != 0 { - log.Infof("Add service containers:") - } - for _, image := range m.Services { - log.Infof(" Create OCI config for %s", image.Image) - useTrust := enforceContentTrust(image.Image, &m.Trust) - config, err := ConfigToOCI(image, useTrust) - if err != nil { - return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err) - } - path := "containers/services/" + image.Name - err = ImageBundle(path, image.Image, config, iw, useTrust, pull) - if err != nil { - return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err) - } - } - - // add files - err := filesystem(m, iw) - if err != nil { - return fmt.Errorf("failed to add filesystem parts: %v", err) - } - - // add anything additional for this output type - if addition != nil { - err = addition(iw) - if err != nil { - return fmt.Errorf("Failed to add additional files: %v", err) - } - } - - err = iw.Close() - if err != nil { - return fmt.Errorf("initrd close error: %v", err) - } - - return nil -} - -// kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer -type kernelFilter struct { - tw *tar.Writer - buffer *bytes.Buffer - cmdline string - discard bool - foundKernel bool - foundKTar bool -} - -func newKernelFilter(tw *tar.Writer, cmdline string) *kernelFilter { - return &kernelFilter{tw: tw, cmdline: cmdline} -} - -func (k *kernelFilter) finishTar() error { - if k.buffer == nil { - return nil - } - tr := tar.NewReader(k.buffer) - err := tarAppend(k.tw, tr) - k.buffer = nil - return err -} - -func (k *kernelFilter) Close() error { - if !k.foundKernel { - return errors.New("did not find kernel in kernel image") - } - if !k.foundKTar { - return errors.New("did not find kernel.tar in kernel image") - } - return k.finishTar() -} - -func (k *kernelFilter) Flush() error { - err := k.finishTar() - if err != nil { - return err - } - return k.tw.Flush() -} - -func (k *kernelFilter) Write(b []byte) (n int, err error) { - if k.discard { - return len(b), nil - } - if k.buffer != nil { - return k.buffer.Write(b) - } - return k.tw.Write(b) -} - -func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { - err := k.finishTar() - if err != nil { - return err - } - tw := k.tw - switch hdr.Name { - case "kernel": - if k.foundKernel { - return errors.New("found more than one possible kernel image") - } - k.foundKernel = true - k.discard = false - whdr := &tar.Header{ - Name: "boot", - Mode: 0755, - Typeflag: tar.TypeDir, - } - if err := tw.WriteHeader(whdr); err != nil { - return err - } - // add the cmdline in /boot/cmdline - whdr = &tar.Header{ - Name: "boot/cmdline", - Mode: 0644, - Size: int64(len(k.cmdline)), - } - if err := tw.WriteHeader(whdr); err != nil { - return err - } - buf := bytes.NewBufferString(k.cmdline) - _, err = io.Copy(tw, buf) - if err != nil { - return err - } - whdr = &tar.Header{ - Name: "boot/kernel", - Mode: hdr.Mode, - Size: hdr.Size, - } - if err := tw.WriteHeader(whdr); err != nil { - return err - } - case "kernel.tar": - k.foundKTar = true - k.discard = false - k.buffer = new(bytes.Buffer) - default: - k.discard = true - } - - return nil -} - -func tarAppend(iw *tar.Writer, tr *tar.Reader) error { - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - err = iw.WriteHeader(hdr) - if err != nil { - return err - } - _, err = io.Copy(iw, tr) - if err != nil { - return err - } - } - return nil -} - -func filesystem(m Moby, tw *tar.Writer) error { - // TODO also include the files added in other parts of the build - var addedFiles = map[string]bool{} - - if len(m.Files) != 0 { - log.Infof("Add files:") - } - for _, f := range m.Files { - log.Infof(" %s", f.Path) - if f.Path == "" { - return errors.New("Did not specify path for file") - } - // tar archives should not have absolute paths - if f.Path[0] == os.PathSeparator { - f.Path = f.Path[1:] - } - mode := int64(0600) - if f.Directory { - mode = 0700 - } - if f.Mode != "" { - var err error - mode, err = strconv.ParseInt(f.Mode, 8, 32) - if err != nil { - return fmt.Errorf("Cannot parse file mode as octal value: %v", err) - } - } - dirMode := mode - if dirMode&0700 != 0 { - dirMode |= 0100 - } - if dirMode&0070 != 0 { - dirMode |= 0010 - } - if dirMode&0007 != 0 { - dirMode |= 0001 - } - var contents []byte - if f.Contents != nil { - contents = []byte(*f.Contents) - } - if !f.Directory && f.Contents == nil && f.Symlink == "" { - if f.Source == "" { - return errors.New("Contents of file not specified") - } - if len(f.Source) > 2 && f.Source[:2] == "~/" { - f.Source = homeDir() + f.Source[1:] - } - if f.Optional { - _, err := os.Stat(f.Source) - if err != nil { - // skip if not found or readable - log.Debugf("Skipping file [%s] as not readable and marked optional", f.Source) - continue - } - } - var err error - contents, err = ioutil.ReadFile(f.Source) - if err != nil { - return err - } - } - // we need all the leading directories - parts := strings.Split(path.Dir(f.Path), "/") - root := "" - for _, p := range parts { - if p == "." || p == "/" { - continue - } - if root == "" { - root = p - } else { - root = root + "/" + p - } - if !addedFiles[root] { - hdr := &tar.Header{ - Name: root, - Typeflag: tar.TypeDir, - Mode: dirMode, - } - err := tw.WriteHeader(hdr) - if err != nil { - return err - } - addedFiles[root] = true - } - } - addedFiles[f.Path] = true - if f.Directory { - if f.Contents != nil { - return errors.New("Directory with contents not allowed") - } - hdr := &tar.Header{ - Name: f.Path, - Typeflag: tar.TypeDir, - Mode: mode, - } - err := tw.WriteHeader(hdr) - if err != nil { - return err - } - } else if f.Symlink != "" { - hdr := &tar.Header{ - Name: f.Path, - Typeflag: tar.TypeSymlink, - Mode: mode, - Linkname: f.Symlink, - } - err := tw.WriteHeader(hdr) - if err != nil { - return err - } - } else { - hdr := &tar.Header{ - Name: f.Path, - Mode: mode, - Size: int64(len(contents)), - } - err := tw.WriteHeader(hdr) - if err != nil { - return err - } - _, err = tw.Write(contents) - if err != nil { - return err - } - } - } - return nil -} diff --git a/cmd/moby/main.go b/cmd/moby/main.go index e4bd2a245..91befa2dc 100644 --- a/cmd/moby/main.go +++ b/cmd/moby/main.go @@ -7,6 +7,7 @@ import ( "path/filepath" log "github.com/Sirupsen/logrus" + "github.com/moby/tool/src/moby" ) var ( @@ -17,9 +18,6 @@ var ( // GitCommit hash, set at compile time GitCommit = "unknown" - - // MobyDir is the location of the cache directory ~/.moby by default - MobyDir string ) // infoFormatter overrides the default format for Info() log events to @@ -91,16 +89,17 @@ func main() { os.Exit(1) } - MobyDir = *flagConfigDir - err := os.MkdirAll(MobyDir, 0755) + mobyDir := *flagConfigDir + err := os.MkdirAll(mobyDir, 0755) if err != nil { - log.Fatalf("Could not create config directory [%s]: %v", MobyDir, err) + log.Fatalf("Could not create config directory [%s]: %v", mobyDir, err) } - err = os.MkdirAll(filepath.Join(MobyDir, "tmp"), 0755) + err = os.MkdirAll(filepath.Join(mobyDir, "tmp"), 0755) if err != nil { - log.Fatalf("Could not create config tmp directory [%s]: %v", filepath.Join(MobyDir, "tmp"), err) + log.Fatalf("Could not create config tmp directory [%s]: %v", filepath.Join(mobyDir, "tmp"), err) } + moby.MobyDir = mobyDir switch args[0] { case "build": diff --git a/src/moby/build.go b/src/moby/build.go new file mode 100644 index 000000000..10acc5d04 --- /dev/null +++ b/src/moby/build.go @@ -0,0 +1,467 @@ +package moby + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "sort" + "strconv" + "strings" + + log "github.com/Sirupsen/logrus" +) + +const defaultNameForStdin = "moby" + +var streamable = map[string]bool{ + "docker": true, + "tar": true, +} + +// Streamable returns true if an output can be streamed +func Streamable(t string) bool { + return streamable[t] +} + +type addFun func(*tar.Writer) error + +const dockerfile = ` +FROM scratch + +COPY . ./ +RUN rm -f Dockerfile + +ENTRYPOINT ["/sbin/tini", "--", "/bin/rc.init"] +` + +var additions = map[string]addFun{ + "docker": func(tw *tar.Writer) error { + log.Infof(" Adding Dockerfile") + hdr := &tar.Header{ + Name: "Dockerfile", + Mode: 0644, + Size: int64(len(dockerfile)), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write([]byte(dockerfile)); err != nil { + return err + } + return nil + }, +} + +// OutputTypes returns a list of the valid output types +func OutputTypes() []string { + ts := []string{} + for k := range streamable { + ts = append(ts, k) + } + for k := range outFuns { + ts = append(ts, k) + } + sort.Strings(ts) + + return ts +} + +func enforceContentTrust(fullImageName string, config *TrustConfig) bool { + for _, img := range config.Image { + // First check for an exact name match + if img == fullImageName { + return true + } + // Also check for an image name only match + // by removing a possible tag (with possibly added digest): + imgAndTag := strings.Split(fullImageName, ":") + if len(imgAndTag) >= 2 && img == imgAndTag[0] { + return true + } + // and by removing a possible digest: + imgAndDigest := strings.Split(fullImageName, "@sha256:") + if len(imgAndDigest) >= 2 && img == imgAndDigest[0] { + return true + } + } + + for _, org := range config.Org { + var imgOrg string + splitName := strings.Split(fullImageName, "/") + switch len(splitName) { + case 0: + // if the image is empty, return false + return false + case 1: + // for single names like nginx, use library + imgOrg = "library" + case 2: + // for names that assume docker hub, like linxukit/alpine, take the first split + imgOrg = splitName[0] + default: + // for names that include the registry, the second piece is the org, ex: docker.io/library/alpine + imgOrg = splitName[1] + } + if imgOrg == org { + return true + } + } + return false +} + +// Build performs the actual build process +func Build(m Moby, w io.Writer, pull bool, tp string) error { + if MobyDir == "" { + return fmt.Errorf("MobyDir for temporary storage not set") + } + + iw := tar.NewWriter(w) + + // add additions + addition := additions[tp] + + if m.Kernel.Image != "" { + // get kernel and initrd tarball from container + log.Infof("Extract kernel image: %s", m.Kernel.Image) + kf := newKernelFilter(iw, m.Kernel.Cmdline) + err := ImageTar(m.Kernel.Image, "", kf, enforceContentTrust(m.Kernel.Image, &m.Trust), pull) + if err != nil { + return fmt.Errorf("Failed to extract kernel image and tarball: %v", err) + } + err = kf.Close() + if err != nil { + return fmt.Errorf("Close error: %v", err) + } + } + + // convert init images to tarballs + if len(m.Init) != 0 { + log.Infof("Add init containers:") + } + for _, ii := range m.Init { + log.Infof("Process init image: %s", ii) + err := ImageTar(ii, "", iw, enforceContentTrust(ii, &m.Trust), pull) + if err != nil { + return fmt.Errorf("Failed to build init tarball from %s: %v", ii, err) + } + } + + if len(m.Onboot) != 0 { + log.Infof("Add onboot containers:") + } + for i, image := range m.Onboot { + log.Infof(" Create OCI config for %s", image.Image) + useTrust := enforceContentTrust(image.Image, &m.Trust) + config, err := ConfigToOCI(image, useTrust) + if err != nil { + return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err) + } + so := fmt.Sprintf("%03d", i) + path := "containers/onboot/" + so + "-" + image.Name + err = ImageBundle(path, image.Image, config, iw, useTrust, pull) + if err != nil { + return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err) + } + } + + if len(m.Services) != 0 { + log.Infof("Add service containers:") + } + for _, image := range m.Services { + log.Infof(" Create OCI config for %s", image.Image) + useTrust := enforceContentTrust(image.Image, &m.Trust) + config, err := ConfigToOCI(image, useTrust) + if err != nil { + return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err) + } + path := "containers/services/" + image.Name + err = ImageBundle(path, image.Image, config, iw, useTrust, pull) + if err != nil { + return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err) + } + } + + // add files + err := filesystem(m, iw) + if err != nil { + return fmt.Errorf("failed to add filesystem parts: %v", err) + } + + // add anything additional for this output type + if addition != nil { + err = addition(iw) + if err != nil { + return fmt.Errorf("Failed to add additional files: %v", err) + } + } + + err = iw.Close() + if err != nil { + return fmt.Errorf("initrd close error: %v", err) + } + + return nil +} + +// kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer +type kernelFilter struct { + tw *tar.Writer + buffer *bytes.Buffer + cmdline string + discard bool + foundKernel bool + foundKTar bool +} + +func newKernelFilter(tw *tar.Writer, cmdline string) *kernelFilter { + return &kernelFilter{tw: tw, cmdline: cmdline} +} + +func (k *kernelFilter) finishTar() error { + if k.buffer == nil { + return nil + } + tr := tar.NewReader(k.buffer) + err := tarAppend(k.tw, tr) + k.buffer = nil + return err +} + +func (k *kernelFilter) Close() error { + if !k.foundKernel { + return errors.New("did not find kernel in kernel image") + } + if !k.foundKTar { + return errors.New("did not find kernel.tar in kernel image") + } + return k.finishTar() +} + +func (k *kernelFilter) Flush() error { + err := k.finishTar() + if err != nil { + return err + } + return k.tw.Flush() +} + +func (k *kernelFilter) Write(b []byte) (n int, err error) { + if k.discard { + return len(b), nil + } + if k.buffer != nil { + return k.buffer.Write(b) + } + return k.tw.Write(b) +} + +func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { + err := k.finishTar() + if err != nil { + return err + } + tw := k.tw + switch hdr.Name { + case "kernel": + if k.foundKernel { + return errors.New("found more than one possible kernel image") + } + k.foundKernel = true + k.discard = false + whdr := &tar.Header{ + Name: "boot", + Mode: 0755, + Typeflag: tar.TypeDir, + } + if err := tw.WriteHeader(whdr); err != nil { + return err + } + // add the cmdline in /boot/cmdline + whdr = &tar.Header{ + Name: "boot/cmdline", + Mode: 0644, + Size: int64(len(k.cmdline)), + } + if err := tw.WriteHeader(whdr); err != nil { + return err + } + buf := bytes.NewBufferString(k.cmdline) + _, err = io.Copy(tw, buf) + if err != nil { + return err + } + whdr = &tar.Header{ + Name: "boot/kernel", + Mode: hdr.Mode, + Size: hdr.Size, + } + if err := tw.WriteHeader(whdr); err != nil { + return err + } + case "kernel.tar": + k.foundKTar = true + k.discard = false + k.buffer = new(bytes.Buffer) + default: + k.discard = true + } + + return nil +} + +func tarAppend(iw *tar.Writer, tr *tar.Reader) error { + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + err = iw.WriteHeader(hdr) + if err != nil { + return err + } + _, err = io.Copy(iw, tr) + if err != nil { + return err + } + } + return nil +} + +func filesystem(m Moby, tw *tar.Writer) error { + // TODO also include the files added in other parts of the build + var addedFiles = map[string]bool{} + + if len(m.Files) != 0 { + log.Infof("Add files:") + } + for _, f := range m.Files { + log.Infof(" %s", f.Path) + if f.Path == "" { + return errors.New("Did not specify path for file") + } + // tar archives should not have absolute paths + if f.Path[0] == os.PathSeparator { + f.Path = f.Path[1:] + } + mode := int64(0600) + if f.Directory { + mode = 0700 + } + if f.Mode != "" { + var err error + mode, err = strconv.ParseInt(f.Mode, 8, 32) + if err != nil { + return fmt.Errorf("Cannot parse file mode as octal value: %v", err) + } + } + dirMode := mode + if dirMode&0700 != 0 { + dirMode |= 0100 + } + if dirMode&0070 != 0 { + dirMode |= 0010 + } + if dirMode&0007 != 0 { + dirMode |= 0001 + } + var contents []byte + if f.Contents != nil { + contents = []byte(*f.Contents) + } + if !f.Directory && f.Contents == nil && f.Symlink == "" { + if f.Source == "" { + return errors.New("Contents of file not specified") + } + if len(f.Source) > 2 && f.Source[:2] == "~/" { + f.Source = homeDir() + f.Source[1:] + } + if f.Optional { + _, err := os.Stat(f.Source) + if err != nil { + // skip if not found or readable + log.Debugf("Skipping file [%s] as not readable and marked optional", f.Source) + continue + } + } + var err error + contents, err = ioutil.ReadFile(f.Source) + if err != nil { + return err + } + } + // we need all the leading directories + parts := strings.Split(path.Dir(f.Path), "/") + root := "" + for _, p := range parts { + if p == "." || p == "/" { + continue + } + if root == "" { + root = p + } else { + root = root + "/" + p + } + if !addedFiles[root] { + hdr := &tar.Header{ + Name: root, + Typeflag: tar.TypeDir, + Mode: dirMode, + } + err := tw.WriteHeader(hdr) + if err != nil { + return err + } + addedFiles[root] = true + } + } + addedFiles[f.Path] = true + if f.Directory { + if f.Contents != nil { + return errors.New("Directory with contents not allowed") + } + hdr := &tar.Header{ + Name: f.Path, + Typeflag: tar.TypeDir, + Mode: mode, + } + err := tw.WriteHeader(hdr) + if err != nil { + return err + } + } else if f.Symlink != "" { + hdr := &tar.Header{ + Name: f.Path, + Typeflag: tar.TypeSymlink, + Mode: mode, + Linkname: f.Symlink, + } + err := tw.WriteHeader(hdr) + if err != nil { + return err + } + } else { + hdr := &tar.Header{ + Name: f.Path, + Mode: mode, + Size: int64(len(contents)), + } + err := tw.WriteHeader(hdr) + if err != nil { + return err + } + _, err = tw.Write(contents) + if err != nil { + return err + } + } + } + return nil +} diff --git a/cmd/moby/config.go b/src/moby/config.go similarity index 97% rename from cmd/moby/config.go rename to src/moby/config.go index d3398ff4d..6e37f5b54 100644 --- a/cmd/moby/config.go +++ b/src/moby/config.go @@ -1,4 +1,4 @@ -package main +package moby import ( "encoding/json" @@ -24,8 +24,8 @@ type Moby struct { Cmdline string } Init []string - Onboot []MobyImage - Services []MobyImage + Onboot []Image + Services []Image Trust TrustConfig Files []struct { Path string @@ -44,8 +44,8 @@ type TrustConfig struct { Org []string } -// MobyImage is the type of an image config -type MobyImage struct { +// Image is the type of an image config +type Image struct { Name string `yaml:"name" json:"name"` Image string `yaml:"image" json:"image"` Capabilities *[]string `yaml:"capabilities" json:"capabilities,omitempty"` @@ -153,11 +153,11 @@ func AppendConfig(m0, m1 Moby) Moby { return moby } -// NewImage validates an parses yaml or json for a MobyImage -func NewImage(config []byte) (MobyImage, error) { +// NewImage validates an parses yaml or json for a Image +func NewImage(config []byte) (Image, error) { log.Debugf("Reading label config: %s", string(config)) - mi := MobyImage{} + mi := Image{} // Parse raw yaml var rawYaml interface{} @@ -220,7 +220,7 @@ func NewImage(config []byte) (MobyImage, error) { } // ConfigToOCI converts a config specification to an OCI config file -func ConfigToOCI(image MobyImage, trust bool) ([]byte, error) { +func ConfigToOCI(image Image, trust bool) ([]byte, error) { // TODO pass through same docker client to all functions cli, err := dockerClient() @@ -421,7 +421,7 @@ func assignStringEmpty4(v1, v2, v3, v4 string) string { } // ConfigInspectToOCI converts a config and the output of image inspect to an OCI config -func ConfigInspectToOCI(yaml MobyImage, inspect types.ImageInspect) (specs.Spec, error) { +func ConfigInspectToOCI(yaml Image, inspect types.ImageInspect) (specs.Spec, error) { oci := specs.Spec{} var inspectConfig container.Config @@ -430,7 +430,7 @@ func ConfigInspectToOCI(yaml MobyImage, inspect types.ImageInspect) (specs.Spec, } // look for org.mobyproject.config label - var label MobyImage + var label Image labelString := inspectConfig.Labels["org.mobyproject.config"] if labelString != "" { var err error diff --git a/cmd/moby/config_test.go b/src/moby/config_test.go similarity index 94% rename from cmd/moby/config_test.go rename to src/moby/config_test.go index bc47860e5..498ca4213 100644 --- a/cmd/moby/config_test.go +++ b/src/moby/config_test.go @@ -1,4 +1,4 @@ -package main +package moby import ( "encoding/json" @@ -12,7 +12,7 @@ import ( func TestOverrides(t *testing.T) { var yamlCaps = []string{"CAP_SYS_ADMIN"} - var yaml = MobyImage{ + var yaml = Image{ Name: "test", Image: "testimage", Capabilities: &yamlCaps, @@ -20,7 +20,7 @@ func TestOverrides(t *testing.T) { var labelCaps = []string{"CAP_SYS_CHROOT"} - var label = MobyImage{ + var label = Image{ Capabilities: &labelCaps, Cwd: "/label/directory", } diff --git a/cmd/moby/docker.go b/src/moby/docker.go similarity index 99% rename from cmd/moby/docker.go rename to src/moby/docker.go index 821c9c92b..5056da17d 100644 --- a/cmd/moby/docker.go +++ b/src/moby/docker.go @@ -1,4 +1,4 @@ -package main +package moby // We want to replace much of this with use of containerd tools // and also using the Docker API not shelling out diff --git a/cmd/moby/image.go b/src/moby/image.go similarity index 99% rename from cmd/moby/image.go rename to src/moby/image.go index 433a64408..69d6ece43 100644 --- a/cmd/moby/image.go +++ b/src/moby/image.go @@ -1,4 +1,4 @@ -package main +package moby import ( "archive/tar" diff --git a/cmd/moby/linuxkit.go b/src/moby/linuxkit.go similarity index 98% rename from cmd/moby/linuxkit.go rename to src/moby/linuxkit.go index e2e8bf043..529f30ad9 100644 --- a/cmd/moby/linuxkit.go +++ b/src/moby/linuxkit.go @@ -1,4 +1,4 @@ -package main +package moby import ( "bytes" @@ -60,7 +60,7 @@ func ensureLinuxkitImage(name string) error { } // TODO pass through --pull to here buf := new(bytes.Buffer) - buildInternal(m, buf, false, nil) + Build(m, buf, false, "") image := buf.Bytes() kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { diff --git a/cmd/moby/output.go b/src/moby/output.go similarity index 96% rename from cmd/moby/output.go rename to src/moby/output.go index 1553790f9..b683cfe2f 100644 --- a/cmd/moby/output.go +++ b/src/moby/output.go @@ -1,4 +1,4 @@ -package main +package moby import ( "archive/tar" @@ -128,7 +128,8 @@ func ensurePrereq(out string) error { return err } -func validateOutputs(out outputList) error { +// ValidateOutputs checks if the output type is known +func ValidateOutputs(out []string) error { log.Debugf("validating output: %v", out) for _, o := range out { @@ -145,10 +146,11 @@ func validateOutputs(out outputList) error { return nil } -func outputs(base string, image []byte, out outputList, size int, hyperkit bool) error { +// Outputs generates all the specified output formats +func Outputs(base string, image []byte, out []string, size int, hyperkit bool) error { log.Debugf("output: %v %s", out, base) - err := validateOutputs(out) + err := ValidateOutputs(out) if err != nil { return err } diff --git a/cmd/moby/schema.go b/src/moby/schema.go similarity index 99% rename from cmd/moby/schema.go rename to src/moby/schema.go index 6cfc48fb9..92ed70c76 100644 --- a/cmd/moby/schema.go +++ b/src/moby/schema.go @@ -1,4 +1,4 @@ -package main +package moby var schema = string(` { diff --git a/cmd/moby/trust.go b/src/moby/trust.go similarity index 99% rename from cmd/moby/trust.go rename to src/moby/trust.go index 145075c7a..b5de5b9a3 100644 --- a/cmd/moby/trust.go +++ b/src/moby/trust.go @@ -1,4 +1,4 @@ -package main +package moby import ( "crypto/tls" diff --git a/cmd/moby/trust_test.go b/src/moby/trust_test.go similarity index 99% rename from cmd/moby/trust_test.go rename to src/moby/trust_test.go index 36d1f2da8..e5d8d7693 100644 --- a/cmd/moby/trust_test.go +++ b/src/moby/trust_test.go @@ -1,4 +1,4 @@ -package main +package moby import "testing" diff --git a/src/moby/util.go b/src/moby/util.go new file mode 100644 index 000000000..b262e0066 --- /dev/null +++ b/src/moby/util.go @@ -0,0 +1,6 @@ +package moby + +var ( + // MobyDir is the location of the cache directory which should be set by the caller + MobyDir string +) diff --git a/src/moby/util_unix.go b/src/moby/util_unix.go new file mode 100644 index 000000000..a22b8579a --- /dev/null +++ b/src/moby/util_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package moby + +import ( + "os" +) + +func homeDir() string { + return os.Getenv("HOME") +} diff --git a/src/moby/util_windows.go b/src/moby/util_windows.go new file mode 100644 index 000000000..386e7e478 --- /dev/null +++ b/src/moby/util_windows.go @@ -0,0 +1,9 @@ +package moby + +import ( + "os" +) + +func homeDir() string { + return os.Getenv("USERPROFILE") +}