Move Go code to src/cmd

This does not get everything where we want it finally, see #1266 nor the optimal way of building, but it gets it out of top level. Added instructions to build if you have a Go installation. Not moving `vendor` yet. Signed-off-by: Justin Cormack <justin.cormack@docker.com>
2026-01-23 07:06:29 +00:00 · 2017-03-21 13:55:52 +00:00
parent 26b4acd135
commit 4dd51b4ada
8 changed files with 8 additions and 4 deletions
--- a/src/cmd/moby/config.go
+++ b/src/cmd/moby/config.go
@@ -0,0 +1,162 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Moby is the type of a Moby config file
+type Moby struct {
+	Kernel struct {
+		Image   string
+		Cmdline string
+	}
+	Init   string
+	System []MobyImage
+	Daemon []MobyImage
+	Files  []struct {
+		Path     string
+		Contents string
+	}
+	Outputs []struct {
+		Format  string
+		Project string
+		Bucket  string
+		Family  string
+		Public  bool
+		Replace bool
+	}
+}
+
+// MobyImage is the type of an image config, based on Compose
+type MobyImage struct {
+	Name         string
+	Image        string
+	Capabilities []string
+	Binds        []string
+	OomScoreAdj  int64 `yaml:"oom_score_adj"`
+	Command      []string
+	NetworkMode  string `yaml:"network_mode"`
+	Pid          string
+	Ipc          string
+	Uts          string
+	ReadOnly     bool `yaml:"read_only"`
+}
+
+const riddler = "mobylinux/riddler:c23ab4b6e2a2a4ebd4dd51a059cef7f270da72cb@sha256:7e7744b2f554518411633200db98e599782b120e323348495f43f540de26f7b6"
+
+// NewConfig parses a config file
+func NewConfig(config []byte) (*Moby, error) {
+	m := Moby{}
+
+	err := yaml.Unmarshal(config, &m)
+	if err != nil {
+		return &m, err
+	}
+
+	return &m, nil
+}
+
+// ConfigToRun converts a config to a series of arguments for docker run
+func ConfigToRun(order int, path string, image *MobyImage) []string {
+	// riddler arguments
+	so := fmt.Sprintf("%03d", order)
+	args := []string{"-v", "/var/run/docker.sock:/var/run/docker.sock", riddler, image.Image, "/containers/" + path + "/" + so + "-" + image.Name}
+	// docker arguments
+	args = append(args, "--cap-drop", "all")
+	for _, cap := range image.Capabilities {
+		if strings.ToUpper(cap)[0:4] == "CAP_" {
+			cap = cap[4:]
+		}
+		args = append(args, "--cap-add", cap)
+	}
+	if image.OomScoreAdj != 0 {
+		args = append(args, "--oom-score-adj", strconv.FormatInt(image.OomScoreAdj, 10))
+	}
+	if image.NetworkMode != "" {
+		// TODO only "host" supported
+		args = append(args, "--net="+image.NetworkMode)
+	}
+	if image.Pid != "" {
+		// TODO only "host" supported
+		args = append(args, "--pid="+image.Pid)
+	}
+	if image.Ipc != "" {
+		// TODO only "host" supported
+		args = append(args, "--ipc="+image.Ipc)
+	}
+	if image.Uts != "" {
+		// TODO only "host" supported
+		args = append(args, "--uts="+image.Uts)
+	}
+	for _, bind := range image.Binds {
+		args = append(args, "-v", bind)
+	}
+	if image.ReadOnly {
+		args = append(args, "--read-only")
+	}
+	// image
+	args = append(args, image.Image)
+	// command
+	args = append(args, image.Command...)
+
+	return args
+}
+
+func filesystem(m *Moby) (*bytes.Buffer, error) {
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	defer tw.Close()
+
+	for _, f := range m.Files {
+		if f.Path == "" {
+			return buf, errors.New("Did not specify path for file")
+		}
+		if f.Contents == "" {
+			return buf, errors.New("Contents of file not specified")
+		}
+		// we need all the leading directories
+		parts := strings.Split(path.Dir(f.Path), "/")
+		root := ""
+		for _, p := range parts {
+			if p == "." || p == "/" {
+				continue
+			}
+			if root == "" {
+				root = p
+			} else {
+				root = root + "/" + p
+			}
+			hdr := &tar.Header{
+				Name:     root,
+				Typeflag: tar.TypeDir,
+				Mode:     0700,
+			}
+			err := tw.WriteHeader(hdr)
+			if err != nil {
+				return buf, err
+			}
+		}
+		hdr := &tar.Header{
+			Name: f.Path,
+			Mode: 0600,
+			Size: int64(len(f.Contents)),
+		}
+		err := tw.WriteHeader(hdr)
+		if err != nil {
+			return buf, err
+		}
+		_, err = tw.Write([]byte(f.Contents))
+		if err != nil {
+			return buf, err
+		}
+	}
+	return buf, nil
+}
--- a/src/cmd/moby/gcp.go
+++ b/src/cmd/moby/gcp.go
@@ -0,0 +1,100 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+
+	"cloud.google.com/go/storage"
+	"golang.org/x/net/context"
+)
+
+func uploadGS(filename, project, bucket string, public bool) error {
+	if project != "" {
+		err := os.Setenv("GOOGLE_CLOUD_PROJECT", project)
+		if err != nil {
+			return err
+		}
+	}
+	if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" {
+		return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config")
+	}
+
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	obj := client.Bucket(bucket).Object(filename)
+	wc := obj.NewWriter(ctx)
+	_, err = io.Copy(wc, f)
+	if err != nil {
+		return err
+	}
+	err = wc.Close()
+	if err != nil {
+		return err
+	}
+
+	if public {
+		err = obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader)
+		if err != nil {
+			return err
+		}
+	}
+
+	fmt.Println("gs://" + bucket + "/" + filename)
+
+	return nil
+}
+
+func imageGS(filename, project, storage, family string, replace bool) error {
+	if project != "" {
+		err := os.Setenv("GOOGLE_CLOUD_PROJECT", project)
+		if err != nil {
+			return err
+		}
+	}
+	if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" {
+		return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config")
+	}
+
+	// TODO do not shell out to gcloud tool, use the API
+
+	gcloud, err := exec.LookPath("gcloud")
+	if err != nil {
+		return errors.New("Please install the gcloud binary")
+	}
+
+	if replace {
+		args := []string{"compute", "images", "delete", filename}
+		cmd := exec.Command(gcloud, args...)
+		// ignore failures; it may not exist
+		_ = cmd.Run()
+	}
+
+	args := []string{"compute", "images", "create", "--source-uri", storage}
+	if family != "" {
+		args = append(args, "--family", family)
+	}
+	args = append(args, filename)
+	cmd := exec.Command(gcloud, args...)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Image creation failed: %v - %s", err, string(out))
+	}
+
+	fmt.Println(filename)
+
+	return nil
+}
--- a/src/cmd/moby/main.go
+++ b/src/cmd/moby/main.go
@@ -0,0 +1,258 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/docker/moby/src/initrd"
+)
+
+const (
+	docker2tar = "mobylinux/docker2tar:82a3f11f70b2959c7100dd6e184b511ebfc65908@sha256:e4fd36febc108477a2e5316d263ac257527779409891c7ac10d455a162df05c1"
+)
+
+func dockerRun(args ...string) ([]byte, error) {
+	// TODO switch to using Docker client API not exec - just a quick prototype
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args = append([]string{"run", "--rm"}, args...)
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func dockerRunInput(input io.Reader, args ...string) ([]byte, error) {
+	// TODO switch to using Docker client API not exec - just a quick prototype
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args = append([]string{"run", "--rm", "-i"}, args...)
+	cmd := exec.Command(docker, args...)
+	cmd.Stdin = input
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
+	tr := tar.NewReader(buf)
+
+	var bzimage, ktar *bytes.Buffer
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalln(err)
+		}
+		switch hdr.Name {
+		case bzimageName:
+			bzimage = new(bytes.Buffer)
+			_, err := io.Copy(bzimage, tr)
+			if err != nil {
+				return nil, nil, err
+			}
+		case ktarName:
+			ktar = new(bytes.Buffer)
+			_, err := io.Copy(bzimage, tr)
+			if err != nil {
+				return nil, nil, err
+			}
+		default:
+			continue
+		}
+	}
+
+	if ktar == nil || bzimage == nil {
+		return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
+	}
+
+	return bzimage, ktar, nil
+}
+
+func containersInitrd(containers []*bytes.Buffer) (*bytes.Buffer, error) {
+	w := new(bytes.Buffer)
+	iw := initrd.NewWriter(w)
+	defer iw.Close()
+	for _, file := range containers {
+		_, err := initrd.Copy(iw, file)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return w, nil
+}
+
+func build(m *Moby, name string) {
+	containers := []*bytes.Buffer{}
+
+	// get kernel bzImage and initrd tarball from container
+	// TODO examine contents to see what names they might have
+	const (
+		bzimageName = "bzImage"
+		ktarName    = "kernel.tar"
+	)
+	out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
+	if err != nil {
+		log.Fatalf("Failed to extract kernel image and tarball: %v", err)
+	}
+	buf := bytes.NewBuffer(out)
+	bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
+	if err != nil {
+		log.Fatalf("Could not extract bzImage and kernel filesystem from tarball")
+	}
+	containers = append(containers, ktar)
+
+	// convert init image to tarball
+	init, err := dockerRun("-v", "/var/run/docker.sock:/var/run/docker.sock", docker2tar, m.Init)
+	if err != nil {
+		log.Fatalf("Failed to build init tarball: %v", err)
+	}
+	buffer := bytes.NewBuffer(init)
+	containers = append(containers, buffer)
+
+	for i, image := range m.System {
+		args := ConfigToRun(i, "system", &image)
+		out, err := dockerRun(args...)
+		if err != nil {
+			log.Fatalf("Failed to build container tarball: %v", err)
+		}
+		buffer := bytes.NewBuffer(out)
+		containers = append(containers, buffer)
+	}
+
+	for i, image := range m.Daemon {
+		args := ConfigToRun(i, "daemon", &image)
+		out, err := dockerRun(args...)
+		if err != nil {
+			log.Fatalf("Failed to build container tarball: %v", err)
+		}
+		buffer := bytes.NewBuffer(out)
+		containers = append(containers, buffer)
+	}
+
+	// add files
+	buffer, err = filesystem(m)
+	if err != nil {
+		log.Fatalf("failed to add filesystem parts: %v", err)
+	}
+	containers = append(containers, buffer)
+
+	initrd, err := containersInitrd(containers)
+	if err != nil {
+		log.Fatalf("Failed to make initrd %v", err)
+	}
+
+	err = outputs(m, name, bzimage.Bytes(), initrd.Bytes())
+	if err != nil {
+		log.Fatalf("Error writing outputs: %v", err)
+	}
+}
+
+var (
+	conf string
+	name string
+)
+
+func main() {
+	flag.StringVar(&name, "name", "", "Name to use for output files")
+	flag.Parse()
+
+	conf = "moby.yaml"
+	if len(flag.Args()) > 0 {
+		conf = flag.Args()[0]
+	}
+
+	if name == "" {
+		name = filepath.Base(conf)
+		ext := filepath.Ext(conf)
+		if ext != "" {
+			name = name[:len(name)-len(ext)]
+		}
+	}
+
+	config, err := ioutil.ReadFile(conf)
+	if err != nil {
+		log.Fatalf("Cannot open config file: %v", err)
+	}
+
+	m, err := NewConfig(config)
+	if err != nil {
+		log.Fatalf("Invalid config: %v", err)
+	}
+
+	build(m, name)
+}
--- a/src/cmd/moby/output.go
+++ b/src/cmd/moby/output.go
@@ -0,0 +1,174 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+const (
+	bios = "mobylinux/mkimage-iso-bios:489b1f054a77a8f379d0bfc6cd91639b4db6b67c@sha256:0f058951aac4367d132682aa19eeb5cdcb05600a5d51fe5d0fcbd97b03ae4f87"
+	efi  = "mobylinux/mkimage-iso-efi:d86021840d2180422bd2f59dcff2fcfb5aea6ad1@sha256:00a6dc21073a24763bc667cadd90c42b69cd69579f8036d6d794b42cbb583142"
+	gce  = "mobylinux/mkimage-gce:2039be4e39e855d1845aee188e266bba3f1d2eed@sha256:e12f76003fd9eaa0c6f39f149db5998cf56de42539b989c994893c8344ca69c0"
+	qcow = "mobylinux/mkimage-qcow:9b3632f111675898ed3a22ac71897e735b5a8364@sha256:2132cf3fb593d65f09c8d109d40e1fad138d81485d4750fc29a7f54611d78d35"
+	vhd  = "mobylinux/mkimage-vhd:73c80e433bf717578c507621a84fd58cec27fe95@sha256:0ae1eda2d6592f309977dc4b25cca120cc4e2ee2cc786e88fdc2761c0d49cb14"
+)
+
+func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
+	for _, o := range m.Outputs {
+		switch o.Format {
+		case "kernel+initrd":
+			err := outputKernelInitrd(base, bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "iso-bios":
+			err := outputISO(bios, base+".iso", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "iso-efi":
+			err := outputISO(efi, base+"-efi.iso", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "gce-img":
+			err := outputImg(gce, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "gce-storage":
+			err := outputImg(gce, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+			if o.Bucket == "" {
+				return fmt.Errorf("No bucket specified for GCE output")
+			}
+			err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public)
+			if err != nil {
+				return fmt.Errorf("Error copying to Google Storage: %v", err)
+			}
+		case "gce":
+			err := outputImg(gce, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+			if o.Bucket == "" {
+				return fmt.Errorf("No bucket specified for GCE output")
+			}
+			err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public)
+			if err != nil {
+				return fmt.Errorf("Error copying to Google Storage: %v", err)
+			}
+			err = imageGS(base, o.Project, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace)
+			if err != nil {
+				return fmt.Errorf("Error creating Google Compute Image: %v", err)
+			}
+		case "qcow", "qcow2":
+			err := outputImg(qcow, base+".qcow2", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "vhd":
+			err := outputImg(vhd, base+".vhd", bzimage, initrd, m.Kernel.Cmdline)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
+		case "":
+			return fmt.Errorf("No format specified for output")
+		default:
+			return fmt.Errorf("Unknown output type %s", o.Format)
+		}
+	}
+	return nil
+}
+
+func tarInitrdKernel(bzimage, initrd []byte) (*bytes.Buffer, error) {
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	hdr := &tar.Header{
+		Name: "bzImage",
+		Mode: 0600,
+		Size: int64(len(bzimage)),
+	}
+	err := tw.WriteHeader(hdr)
+	if err != nil {
+		return buf, err
+	}
+	_, err = tw.Write(bzimage)
+	if err != nil {
+		return buf, err
+	}
+	hdr = &tar.Header{
+		Name: "initrd.img",
+		Mode: 0600,
+		Size: int64(len(initrd)),
+	}
+	err = tw.WriteHeader(hdr)
+	if err != nil {
+		return buf, err
+	}
+	_, err = tw.Write(initrd)
+	if err != nil {
+		return buf, err
+	}
+	err = tw.Close()
+	if err != nil {
+		return buf, err
+	}
+	return buf, nil
+}
+
+func outputImg(image, filename string, bzimage []byte, initrd []byte, args ...string) error {
+	buf, err := tarInitrdKernel(bzimage, initrd)
+	if err != nil {
+		return err
+	}
+	img, err := dockerRunInput(buf, append([]string{image}, args...)...)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(filename, img, os.FileMode(0644))
+	if err != nil {
+		return err
+	}
+	fmt.Println(filename)
+	return nil
+}
+
+func outputISO(image, filename string, bzimage []byte, initrd []byte, args ...string) error {
+	buf, err := tarInitrdKernel(bzimage, initrd)
+	if err != nil {
+		return err
+	}
+	iso, err := dockerRunInput(buf, append([]string{image}, args...)...)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(filename, iso, os.FileMode(0644))
+	if err != nil {
+		return err
+	}
+	fmt.Println(filename)
+	return nil
+}
+
+func outputKernelInitrd(base string, bzimage []byte, initrd []byte, cmdline string) error {
+	err := ioutil.WriteFile(base+"-initrd.img", initrd, os.FileMode(0644))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(base+"-bzImage", bzimage, os.FileMode(0644))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(base+"-cmdline", []byte(cmdline), os.FileMode(0644))
+	if err != nil {
+		return err
+	}
+	fmt.Println(base + "-bzImage " + base + "-initrd.img " + base + "-cmdline")
+	return nil
+}