github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-18 17:01:07 +00:00
Code Issues Projects Releases Wiki Activity

Remove the moby tool from this repo

Browse Source 
Updated go-compile to be able to compile remotely. Note I
did not update the oter users of go-compile as it does not affect
them.

Update `go get` instructions to fetch new one, or `make && make install`
will still work.

Signed-off-by: Justin Cormack <justin.cormack@docker.com>
This commit is contained in:
Justin Cormack 2017-04-25 14:52:25 +01:00
parent 35d8070abc
commit 252e32aac5
10 changed files with 31 additions and 1633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -5,7 +5,7 @@ all: default
VERSION="0.0" # dummy for now
GIT_COMMIT=$(shell git rev-list -1 HEAD)
GO_COMPILE=linuxkit/go-compile:4513068d9a7e919e4ec42e2d7ee879ff5b95b7f5@sha256:bdfadbe3e4ec699ca45b67453662321ec270f2d1a1dbdbf09625776d3ebd68c5
GO_COMPILE=linuxkit/go-compile:5bf17af781df44f07906099402680b9a661f999b@sha256:0bf523bcebb96ccc525f983a118f1fd8cb5e17dbf90e83044ca71bb983000e70
MOBY?=bin/moby
LINUXKIT?=bin/linuxkit
@ -17,10 +17,8 @@ endif
PREFIX?=/usr/local/
MOBY_DEPS=$(wildcard src/cmd/moby/*.go) Makefile vendor.conf
MOBY_DEPS+=$(wildcard src/initrd/*.go) $(wildcard src/pad4/*.go)
bin/moby: $(MOBY_DEPS) | bin
tar cf - vendor src/initrd src/pad4 -C src/cmd/moby . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/linuxkit/linuxkit --ldflags "-X main.GitCommit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o $@ > tmp_moby_bin.tar
bin/moby: | bin
docker run --rm --log-driver=none $(CROSS) $(GO_COMPILE) --clone-path github.com/moby/tool --clone https://github.com/moby/tool.git --package github.com/moby/tool/cmd/moby --ldflags "-X main.GitCommit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o $@ > tmp_moby_bin.tar
tar xf tmp_moby_bin.tar > $@
rm tmp_moby_bin.tar
touch $@

2
README.md
View File

@ -22,7 +22,7 @@ LinuxKit uses the `moby` tool for image builds, and the `linuxkit` tool for push
Simple build instructions: use `make` to build. This will build the tools in `bin/`. Add this
to your `PATH` or copy it to somewhere in your `PATH` eg `sudo cp bin/* /usr/local/bin/`. Or you can use `sudo make install`.
If you already have `go` installed you can use `go get -u github.com/linuxkit/linuxkit/src/cmd/moby` to install
If you already have `go` installed you can use `go get -u github.com/moby/tool/cmd/moby` to install
the `moby` build tool, and `go get -u github.com/linuxkit/linuxkit/src/cmd/linuxkit` to install the `linuxkit` tool.
Once you have built the tool, use `moby build linuxkit.yml` to build the example configuration,

248
src/cmd/moby/build.go
View File

@ -1,248 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/linuxkit/linuxkit/src/initrd"
)
// Process the build arguments and execute build
func build(args []string) {
buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
buildCmd.Usage = func() {
fmt.Printf("USAGE: %s build [options] <file>[.yml]\n\n", os.Args[0])
fmt.Printf("Options:\n")
buildCmd.PrintDefaults()
}
buildName := buildCmd.String("name", "", "Name to use for output files")
buildPull := buildCmd.Bool("pull", false, "Always pull images")
if err := buildCmd.Parse(args); err != nil {
log.Fatal("Unable to parse args")
}
remArgs := buildCmd.Args()
if len(remArgs) == 0 {
fmt.Println("Please specify a configuration file")
buildCmd.Usage()
os.Exit(1)
}
conf := remArgs[0]
if !(filepath.Ext(conf) == ".yml" || filepath.Ext(conf) == ".yaml") {
conf = conf + ".yml"
}
buildInternal(*buildName, *buildPull, conf)
}
func initrdAppend(iw *initrd.Writer, r io.Reader) {
_, err := initrd.Copy(iw, r)
if err != nil {
log.Fatalf("initrd write error: %v", err)
}
}
func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
for _, img := range config.Image {
// First check for an exact name match
if img == fullImageName {
return true
}
// Also check for an image name only match
// by removing a possible tag (with possibly added digest):
if img == strings.TrimSuffix(fullImageName, ":") {
return true
}
// and by removing a possible digest:
if img == strings.TrimSuffix(fullImageName, "@sha256:") {
return true
}
}
for _, org := range config.Org {
if strings.HasPrefix(fullImageName, org+"/") {
return true
}
}
return false
}
// Perform the actual build process
func buildInternal(name string, pull bool, conf string) {
if name == "" {
name = filepath.Base(conf)
ext := filepath.Ext(conf)
if ext != "" {
name = name[:len(name)-len(ext)]
}
}
config, err := ioutil.ReadFile(conf)
if err != nil {
log.Fatalf("Cannot open config file: %v", err)
}
m, err := NewConfig(config)
if err != nil {
log.Fatalf("Invalid config: %v", err)
}
w := new(bytes.Buffer)
iw := initrd.NewWriter(w)
if pull || enforceContentTrust(m.Kernel.Image, &m.Trust) {
log.Infof("Pull kernel image: %s", m.Kernel.Image)
err := dockerPull(m.Kernel.Image, enforceContentTrust(m.Kernel.Image, &m.Trust))
if err != nil {
log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
}
}
// get kernel bzImage and initrd tarball from container
// TODO examine contents to see what names they might have
log.Infof("Extract kernel image: %s", m.Kernel.Image)
const (
bzimageName = "bzImage"
ktarName = "kernel.tar"
)
out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
if err != nil {
log.Fatalf("Failed to extract kernel image and tarball: %v", err)
}
buf := bytes.NewBuffer(out)
bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
if err != nil {
log.Fatalf("Could not extract bzImage and kernel filesystem from tarball. %v", err)
}
initrdAppend(iw, ktar)
// convert init images to tarballs
log.Infof("Add init containers:")
for _, ii := range m.Init {
if pull || enforceContentTrust(ii, &m.Trust) {
log.Infof("Pull init image: %s", ii)
err := dockerPull(ii, enforceContentTrust(ii, &m.Trust))
if err != nil {
log.Fatalf("Could not pull image %s: %v", ii, err)
}
}
log.Infof("Process init image: %s", ii)
init, err := ImageExtract(ii, "")
if err != nil {
log.Fatalf("Failed to build init tarball from %s: %v", ii, err)
}
buffer := bytes.NewBuffer(init)
initrdAppend(iw, buffer)
}
log.Infof("Add onboot containers:")
for i, image := range m.Onboot {
if pull || enforceContentTrust(image.Image, &m.Trust) {
log.Infof(" Pull: %s", image.Image)
err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
if err != nil {
log.Fatalf("Could not pull image %s: %v", image.Image, err)
}
}
log.Infof(" Create OCI config for %s", image.Image)
config, err := ConfigToOCI(&image)
if err != nil {
log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
}
so := fmt.Sprintf("%03d", i)
path := "containers/onboot/" + so + "-" + image.Name
out, err := ImageBundle(path, image.Image, config)
if err != nil {
log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
}
buffer := bytes.NewBuffer(out)
initrdAppend(iw, buffer)
}
log.Infof("Add service containers:")
for _, image := range m.Services {
if pull || enforceContentTrust(image.Image, &m.Trust) {
log.Infof(" Pull: %s", image.Image)
err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
if err != nil {
log.Fatalf("Could not pull image %s: %v", image.Image, err)
}
}
log.Infof(" Create OCI config for %s", image.Image)
config, err := ConfigToOCI(&image)
if err != nil {
log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
}
path := "containers/services/" + image.Name
out, err := ImageBundle(path, image.Image, config)
if err != nil {
log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
}
buffer := bytes.NewBuffer(out)
initrdAppend(iw, buffer)
}
// add files
buffer, err := filesystem(m)
if err != nil {
log.Fatalf("failed to add filesystem parts: %v", err)
}
initrdAppend(iw, buffer)
err = iw.Close()
if err != nil {
log.Fatalf("initrd close error: %v", err)
}
log.Infof("Create outputs:")
err = outputs(m, name, bzimage.Bytes(), w.Bytes())
if err != nil {
log.Fatalf("Error writing outputs: %v", err)
}
}
func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
tr := tar.NewReader(buf)
var bzimage, ktar *bytes.Buffer
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalln(err)
}
switch hdr.Name {
case bzimageName:
bzimage = new(bytes.Buffer)
_, err := io.Copy(bzimage, tr)
if err != nil {
return nil, nil, err
}
case ktarName:
ktar = new(bytes.Buffer)
_, err := io.Copy(ktar, tr)
if err != nil {
return nil, nil, err
}
default:
continue
}
}
if ktar == nil || bzimage == nil {
return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
}
return bzimage, ktar, nil
}

490
src/cmd/moby/config.go
View File

@ -1,490 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/docker/engine-api/types"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/xeipuuv/gojsonschema"
"gopkg.in/yaml.v2"
)
// Moby is the type of a Moby config file
type Moby struct {
Kernel struct {
Image string
Cmdline string
}
Init []string
Onboot []MobyImage
Services []MobyImage
Trust TrustConfig
Files []struct {
Path string
Directory bool
Contents string
}
Outputs []struct {
Format string
}
}
// TrustConfig is the type of a content trust config
type TrustConfig struct {
Image []string
Org []string
}
// MobyImage is the type of an image config
type MobyImage struct {
Name string
Image string
Capabilities []string
Mounts []specs.Mount
Binds []string
Tmpfs []string
Command []string
Env []string
Cwd string
Net string
Pid string
Ipc string
Uts string
Readonly bool
MaskedPaths []string `yaml:"maskedPaths"`
ReadonlyPaths []string `yaml:"readonlyPaths"`
UID uint32 `yaml:"uid"`
GID uint32 `yaml:"gid"`
AdditionalGids []uint32 `yaml:"additionalGids"`
NoNewPrivileges bool `yaml:"noNewPrivileges"`
Hostname string
OomScoreAdj int `yaml:"oomScoreAdj"`
DisableOOMKiller bool `yaml:"disableOOMKiller"`
RootfsPropagation string `yaml:"rootfsPropagation"`
CgroupsPath string `yaml:"cgroupsPath"`
Sysctl map[string]string
}
// github.com/go-yaml/yaml treats map keys as interface{} while encoding/json
// requires them to be strings, integers or to implement encoding.TextMarshaler.
// Fix this up by recursively mapping all map[interface{}]interface{} types into
// map[string]interface{}.
// see http://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang#answer-40737676
func convert(i interface{}) interface{} {
switch x := i.(type) {
case map[interface{}]interface{}:
m2 := map[string]interface{}{}
for k, v := range x {
m2[k.(string)] = convert(v)
}
return m2
case []interface{}:
for i, v := range x {
x[i] = convert(v)
}
}
return i
}
// NewConfig parses a config file
func NewConfig(config []byte) (*Moby, error) {
m := Moby{}
// Parse raw yaml
var rawYaml interface{}
err := yaml.Unmarshal(config, &rawYaml)
if err != nil {
return &m, err
}
// Convert to raw JSON
rawJSON := convert(rawYaml)
// Validate raw yaml with JSON schema
schemaLoader := gojsonschema.NewStringLoader(schema)
documentLoader := gojsonschema.NewGoLoader(rawJSON)
result, err := gojsonschema.Validate(schemaLoader, documentLoader)
if err != nil {
return &m, err
}
if !result.Valid() {
fmt.Printf("The configuration file is invalid:\n")
for _, desc := range result.Errors() {
fmt.Printf("- %s\n", desc)
}
return &m, fmt.Errorf("invalid configuration file")
}
// Parse yaml
err = yaml.Unmarshal(config, &m)
if err != nil {
return &m, err
}
return &m, nil
}
// ConfigToOCI converts a config specification to an OCI config file
func ConfigToOCI(image *MobyImage) ([]byte, error) {
// TODO pass through same docker client to all functions
cli, err := dockerClient()
if err != nil {
return []byte{}, err
}
inspect, err := dockerInspectImage(cli, image.Image)
if err != nil {
return []byte{}, err
}
return ConfigInspectToOCI(image, inspect)
}
func defaultMountpoint(tp string) string {
switch tp {
case "proc":
return "/proc"
case "devpts":
return "/dev/pts"
case "sysfs":
return "/sys"
case "cgroup":
return "/sys/fs/cgroup"
case "mqueue":
return "/dev/mqueue"
default:
return ""
}
}
// Sort mounts by number of path components so /dev/pts is listed after /dev
type mlist []specs.Mount
func (m mlist) Len() int {
return len(m)
}
func (m mlist) Less(i, j int) bool {
return m.parts(i) < m.parts(j)
}
func (m mlist) Swap(i, j int) {
m[i], m[j] = m[j], m[i]
}
func (m mlist) parts(i int) int {
return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
}
// ConfigInspectToOCI converts a config and the output of image inspect to an OCI config file
func ConfigInspectToOCI(image *MobyImage, inspect types.ImageInspect) ([]byte, error) {
oci := specs.Spec{}
config := inspect.Config
if config == nil {
return []byte{}, errors.New("empty image config")
}
args := append(config.Entrypoint, config.Cmd...)
if len(image.Command) != 0 {
args = image.Command
}
env := config.Env
if len(image.Env) != 0 {
env = image.Env
}
cwd := config.WorkingDir
if image.Cwd != "" {
cwd = image.Cwd
}
if cwd == "" {
cwd = "/"
}
// default options match what Docker does
procOptions := []string{"nosuid", "nodev", "noexec", "relatime"}
devOptions := []string{"nosuid", "strictatime", "mode=755", "size=65536k"}
if image.Readonly {
devOptions = append(devOptions, "ro")
}
ptsOptions := []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}
sysOptions := []string{"nosuid", "noexec", "nodev"}
if image.Readonly {
sysOptions = append(sysOptions, "ro")
}
cgroupOptions := []string{"nosuid", "noexec", "nodev", "relatime", "ro"}
// note omits "standard" /dev/shm and /dev/mqueue
mounts := map[string]specs.Mount{
"/proc": {Destination: "/proc", Type: "proc", Source: "proc", Options: procOptions},
"/dev": {Destination: "/dev", Type: "tmpfs", Source: "tmpfs", Options: devOptions},
"/dev/pts": {Destination: "/dev/pts", Type: "devpts", Source: "devpts", Options: ptsOptions},
"/sys": {Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: sysOptions},
"/sys/fs/cgroup": {Destination: "/sys/fs/cgroup", Type: "cgroup", Source: "cgroup", Options: cgroupOptions},
}
for _, t := range image.Tmpfs {
parts := strings.Split(t, ":")
if len(parts) > 2 {
return []byte{}, fmt.Errorf("Cannot parse tmpfs, too many ':': %s", t)
}
dest := parts[0]
opts := []string{}
if len(parts) == 2 {
opts = strings.Split(parts[2], ",")
}
mounts[dest] = specs.Mount{Destination: dest, Type: "tmpfs", Source: "tmpfs", Options: opts}
}
for _, b := range image.Binds {
parts := strings.Split(b, ":")
if len(parts) < 2 {
return []byte{}, fmt.Errorf("Cannot parse bind, missing ':': %s", b)
}
if len(parts) > 3 {
return []byte{}, fmt.Errorf("Cannot parse bind, too many ':': %s", b)
}
src := parts[0]
dest := parts[1]
opts := []string{"rw", "rbind", "rprivate"}
if len(parts) == 3 {
opts = strings.Split(parts[2], ",")
}
mounts[dest] = specs.Mount{Destination: dest, Type: "bind", Source: src, Options: opts}
}
for _, m := range image.Mounts {
tp := m.Type
src := m.Source
dest := m.Destination
opts := m.Options
if tp == "" {
switch src {
case "mqueue", "devpts", "proc", "sysfs", "cgroup":
tp = src
}
}
if tp == "" && dest == "/dev" {
tp = "tmpfs"
}
if tp == "" {
return []byte{}, fmt.Errorf("Mount for destination %s is missing type", dest)
}
if src == "" {
// usually sane, eg proc, tmpfs etc
src = tp
}
if dest == "" {
dest = defaultMountpoint(tp)
}
if dest == "" {
return []byte{}, fmt.Errorf("Mount type %s is missing destination", tp)
}
mounts[dest] = specs.Mount{Destination: dest, Type: tp, Source: src, Options: opts}
}
mountList := mlist{}
for _, m := range mounts {
mountList = append(mountList, m)
}
sort.Sort(mountList)
namespaces := []specs.LinuxNamespace{}
// to attach to an existing namespace, easiest to bind mount with nsfs in a system container
if image.Net != "host" {
namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: image.Net})
}
if image.Pid != "host" {
namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace, Path: image.Pid})
}
if image.Ipc != "host" {
namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.IPCNamespace, Path: image.Ipc})
}
if image.Uts != "host" {
namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.UTSNamespace, Path: image.Uts})
}
// TODO user, cgroup namespaces, maybe mount=host if useful
namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.MountNamespace})
caps := image.Capabilities
if len(caps) == 1 && strings.ToLower(caps[0]) == "all" {
caps = []string{
"CAP_AUDIT_CONTROL",
"CAP_AUDIT_READ",
"CAP_AUDIT_WRITE",
"CAP_BLOCK_SUSPEND",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_DAC_READ_SEARCH",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_IPC_LOCK",
"CAP_IPC_OWNER",
"CAP_KILL",
"CAP_LEASE",
"CAP_LINUX_IMMUTABLE",
"CAP_MAC_ADMIN",
"CAP_MAC_OVERRIDE",
"CAP_MKNOD",
"CAP_NET_ADMIN",
"CAP_NET_BIND_SERVICE",
"CAP_NET_BROADCAST",
"CAP_NET_RAW",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYSLOG",
"CAP_SYS_ADMIN",
"CAP_SYS_BOOT",
"CAP_SYS_CHROOT",
"CAP_SYS_MODULE",
"CAP_SYS_NICE",
"CAP_SYS_PACCT",
"CAP_SYS_PTRACE",
"CAP_SYS_RAWIO",
"CAP_SYS_RESOURCE",
"CAP_SYS_TIME",
"CAP_SYS_TTY_CONFIG",
"CAP_WAKE_ALARM",
}
}
oci.Version = specs.Version
oci.Platform = specs.Platform{
OS: inspect.Os,
Arch: inspect.Architecture,
}
oci.Process = specs.Process{
Terminal: false,
//ConsoleSize
User: specs.User{
UID: image.UID,
GID: image.GID,
AdditionalGids: image.AdditionalGids,
// Username (Windows)
},
Args: args,
Env: env,
Cwd: cwd,
Capabilities: &specs.LinuxCapabilities{
Bounding: caps,
Effective: caps,
Inheritable: caps,
Permitted: caps,
Ambient: []string{},
},
Rlimits: []specs.LinuxRlimit{},
NoNewPrivileges: image.NoNewPrivileges,
// ApparmorProfile
// SelinuxLabel
}
oci.Root = specs.Root{
Path: "rootfs",
Readonly: image.Readonly,
}
oci.Hostname = image.Hostname
oci.Mounts = mountList
oci.Linux = &specs.Linux{
// UIDMappings
// GIDMappings
Sysctl: image.Sysctl,
Resources: &specs.LinuxResources{
// Devices
DisableOOMKiller: &image.DisableOOMKiller,
// Memory
// CPU
// Pids
// BlockIO
// HugepageLimits
// Network
},
CgroupsPath: image.CgroupsPath,
Namespaces: namespaces,
// Devices
// Seccomp
RootfsPropagation: image.RootfsPropagation,
MaskedPaths: image.MaskedPaths,
ReadonlyPaths: image.ReadonlyPaths,
// MountLabel
// IntelRdt
}
return json.MarshalIndent(oci, "", " ")
}
func filesystem(m *Moby) (*bytes.Buffer, error) {
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
defer tw.Close()
log.Infof("Add files:")
for _, f := range m.Files {
log.Infof(" %s", f.Path)
if f.Path == "" {
return buf, errors.New("Did not specify path for file")
}
if !f.Directory && f.Contents == "" {
return buf, errors.New("Contents of file not specified")
}
// we need all the leading directories
parts := strings.Split(path.Dir(f.Path), "/")
root := ""
for _, p := range parts {
if p == "." || p == "/" {
continue
}
if root == "" {
root = p
} else {
root = root + "/" + p
}
hdr := &tar.Header{
Name: root,
Typeflag: tar.TypeDir,
Mode: 0700,
}
err := tw.WriteHeader(hdr)
if err != nil {
return buf, err
}
}
if f.Directory {
if f.Contents != "" {
return buf, errors.New("Directory with contents not allowed")
}
hdr := &tar.Header{
Name: f.Path,
Typeflag: tar.TypeDir,
Mode: 0700,
}
err := tw.WriteHeader(hdr)
if err != nil {
return buf, err
}
} else {
hdr := &tar.Header{
Name: f.Path,
Mode: 0600,
Size: int64(len(f.Contents)),
}
err := tw.WriteHeader(hdr)
if err != nil {
return buf, err
}
_, err = tw.Write([]byte(f.Contents))
if err != nil {
return buf, err
}
}
}
return buf, nil
}

318
src/cmd/moby/docker.go
View File

@ -1,318 +0,0 @@
package main
// We want to replace much of this with use of containerd tools
// and also using the Docker API not shelling out
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
"golang.org/x/net/context"
)
func dockerRun(args ...string) ([]byte, error) {
log.Debugf("docker run: %s", strings.Join(args, " "))
docker, err := exec.LookPath("docker")
if err != nil {
return []byte{}, errors.New("Docker does not seem to be installed")
}
args = append([]string{"run", "--rm", "--log-driver=none"}, args...)
cmd := exec.Command(docker, args...)
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return []byte{}, err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return []byte{}, err
}
err = cmd.Start()
if err != nil {
return []byte{}, err
}
stdout, err := ioutil.ReadAll(stdoutPipe)
if err != nil {
return []byte{}, err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return []byte{}, err
}
err = cmd.Wait()
if err != nil {
return []byte{}, fmt.Errorf("%v: %s", err, stderr)
}
log.Debugf("docker run: %s...Done", strings.Join(args, " "))
return stdout, nil
}
func dockerRunInput(input io.Reader, args ...string) ([]byte, error) {
log.Debugf("docker run (input): %s", strings.Join(args, " "))
docker, err := exec.LookPath("docker")
if err != nil {
return []byte{}, errors.New("Docker does not seem to be installed")
}
args = append([]string{"run", "--rm", "-i"}, args...)
cmd := exec.Command(docker, args...)
cmd.Stdin = input
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return []byte{}, err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return []byte{}, err
}
err = cmd.Start()
if err != nil {
return []byte{}, err
}
stdout, err := ioutil.ReadAll(stdoutPipe)
if err != nil {
return []byte{}, err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return []byte{}, err
}
err = cmd.Wait()
if err != nil {
return []byte{}, fmt.Errorf("%v: %s", err, stderr)
}
log.Debugf("docker run (input): %s...Done", strings.Join(args, " "))
return stdout, nil
}
func dockerCreate(image string) (string, error) {
log.Debugf("docker create: %s", image)
docker, err := exec.LookPath("docker")
if err != nil {
return "", errors.New("Docker does not seem to be installed")
}
// we do not ever run the container, so /dev/null is used as command
args := []string{"create", image, "/dev/null"}
cmd := exec.Command(docker, args...)
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return "", err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return "", err
}
err = cmd.Start()
if err != nil {
return "", err
}
stdout, err := ioutil.ReadAll(stdoutPipe)
if err != nil {
return "", err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return "", err
}
err = cmd.Wait()
if err != nil {
return "", fmt.Errorf("%v: %s", err, stderr)
}
container := strings.TrimSpace(string(stdout))
log.Debugf("docker create: %s...Done", image)
return container, nil
}
func dockerExport(container string) ([]byte, error) {
log.Debugf("docker export: %s", container)
docker, err := exec.LookPath("docker")
if err != nil {
return []byte{}, errors.New("Docker does not seem to be installed")
}
args := []string{"export", container}
cmd := exec.Command(docker, args...)
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return []byte{}, err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return []byte{}, err
}
err = cmd.Start()
if err != nil {
return []byte{}, err
}
stdout, err := ioutil.ReadAll(stdoutPipe)
if err != nil {
return []byte{}, err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return []byte{}, err
}
err = cmd.Wait()
if err != nil {
return []byte{}, fmt.Errorf("%v: %s", err, stderr)
}
log.Debugf("docker export: %s...Done", container)
return stdout, nil
}
func dockerRm(container string) error {
log.Debugf("docker rm: %s", container)
docker, err := exec.LookPath("docker")
if err != nil {
return errors.New("Docker does not seem to be installed")
}
args := []string{"rm", container}
cmd := exec.Command(docker, args...)
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = ioutil.ReadAll(stdoutPipe)
if err != nil {
return err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return err
}
err = cmd.Wait()
if err != nil {
return fmt.Errorf("%v: %s", err, stderr)
}
log.Debugf("docker rm: %s...Done", container)
return nil
}
func dockerPull(image string, trustedPull bool) error {
log.Debugf("docker pull: %s", image)
docker, err := exec.LookPath("docker")
if err != nil {
return errors.New("Docker does not seem to be installed")
}
var args = []string{"pull"}
if trustedPull {
log.Debugf("pulling %s with content trust", image)
args = append(args, "--disable-content-trust=false")
}
args = append(args, image)
cmd := exec.Command(docker, args...)
stderrPipe, err := cmd.StderrPipe()
if err != nil {
return err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = ioutil.ReadAll(stdoutPipe)
if err != nil {
return err
}
stderr, err := ioutil.ReadAll(stderrPipe)
if err != nil {
return err
}
err = cmd.Wait()
if err != nil {
return fmt.Errorf("%v: %s", err, stderr)
}
log.Debugf("docker pull: %s...Done", image)
return nil
}
func dockerClient() (*client.Client, error) {
// for maximum compatibility as we use nothing new
err := os.Setenv("DOCKER_API_VERSION", "1.23")
if err != nil {
return nil, err
}
return client.NewEnvClient()
}
func dockerInspectImage(cli *client.Client, image string) (types.ImageInspect, error) {
log.Debugf("docker inspect image: %s", image)
inspect, _, err := cli.ImageInspectWithRaw(context.Background(), image, false)
if err != nil {
if client.IsErrImageNotFound(err) {
pullErr := dockerPull(image, false)
if pullErr != nil {
return types.ImageInspect{}, pullErr
}
inspect, _, err = cli.ImageInspectWithRaw(context.Background(), image, false)
if err != nil {
return types.ImageInspect{}, err
}
} else {
return types.ImageInspect{}, err
}
}
log.Debugf("docker inspect image: %s...Done", image)
return inspect, nil
}

195
src/cmd/moby/image.go
View File

@ -1,195 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"fmt"
"io"
"io/ioutil"
"strings"
log "github.com/Sirupsen/logrus"
)
// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.
var exclude = map[string]bool{
".dockerenv": true,
"Dockerfile": true,
"dev/console": true,
"dev/pts": true,
"dev/shm": true,
}
var replace = map[string]string{
"etc/hosts": `127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`,
"etc/resolv.conf": `nameserver 8.8.8.8
nameserver 8.8.4.4
nameserver 2001:4860:4860::8888
nameserver 2001:4860:4860::8844
`,
"etc/hostname": "moby",
}
// ImageExtract extracts the filesystem from an image and returns a tarball with the files prefixed by the given path
func ImageExtract(image, prefix string) ([]byte, error) {
log.Debugf("image extract: %s %s", image, prefix)
out := new(bytes.Buffer)
tw := tar.NewWriter(out)
err := tarPrefix(prefix, tw)
if err != nil {
return []byte{}, err
}
err = imageTar(image, prefix, tw)
if err != nil {
return []byte{}, err
}
err = tw.Close()
if err != nil {
return []byte{}, err
}
return out.Bytes(), nil
}
// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw *tar.Writer) error {
if path == "" {
return nil
}
if path[len(path)-1] != byte('/') {
return fmt.Errorf("path does not end with /: %s", path)
}
path = path[:len(path)-1]
if path[0] == byte('/') {
return fmt.Errorf("path should be relative: %s", path)
}
mkdir := ""
for _, dir := range strings.Split(path, "/") {
mkdir = mkdir + dir
hdr := &tar.Header{
Name: mkdir,
Mode: 0755,
Typeflag: tar.TypeDir,
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
mkdir = mkdir + "/"
}
return nil
}
func imageTar(image, prefix string, tw *tar.Writer) error {
log.Debugf("image tar: %s %s", image, prefix)
if prefix != "" && prefix[len(prefix)-1] != byte('/') {
return fmt.Errorf("prefix does not end with /: %s", prefix)
}
container, err := dockerCreate(image)
if err != nil {
return fmt.Errorf("Failed to docker create image %s: %v", image, err)
}
contents, err := dockerExport(container)
if err != nil {
return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
}
err = dockerRm(container)
if err != nil {
return fmt.Errorf("Failed to docker rm container %s: %v", container, err)
}
// now we need to filter out some files from the resulting tar archive
r := bytes.NewReader(contents)
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if exclude[hdr.Name] {
log.Debugf("image tar: %s %s exclude %s", image, prefix, hdr.Name)
_, err = io.Copy(ioutil.Discard, tr)
if err != nil {
return err
}
} else if replace[hdr.Name] != "" {
contents := replace[hdr.Name]
hdr.Size = int64(len(contents))
hdr.Name = prefix + hdr.Name
log.Debugf("image tar: %s %s add %s", image, prefix, hdr.Name)
if err := tw.WriteHeader(hdr); err != nil {
return err
}
buf := bytes.NewBufferString(contents)
_, err = io.Copy(tw, buf)
if err != nil {
return err
}
_, err = io.Copy(ioutil.Discard, tr)
if err != nil {
return err
}
} else {
log.Debugf("image tar: %s %s add %s", image, prefix, hdr.Name)
hdr.Name = prefix + hdr.Name
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, tr)
if err != nil {
return err
}
}
}
err = tw.Close()
if err != nil {
return err
}
return nil
}
// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(path string, image string, config []byte) ([]byte, error) {
log.Debugf("image bundle: %s %s cfg: %s", path, image, string(config))
out := new(bytes.Buffer)
tw := tar.NewWriter(out)
err := tarPrefix(path+"/rootfs/", tw)
if err != nil {
return []byte{}, err
}
hdr := &tar.Header{
Name: path + "/" + "config.json",
Mode: 0644,
Size: int64(len(config)),
}
err = tw.WriteHeader(hdr)
if err != nil {
return []byte{}, err
}
buf := bytes.NewBuffer(config)
_, err = io.Copy(tw, buf)
if err != nil {
return []byte{}, err
}
err = imageTar(image, path+"/rootfs/", tw)
if err != nil {
return []byte{}, err
}
err = tw.Close()
if err != nil {
return []byte{}, err
}
return out.Bytes(), nil
}

92
src/cmd/moby/main.go
View File

@ -1,92 +0,0 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
log "github.com/Sirupsen/logrus"
)
var (
defaultLogFormatter = &log.TextFormatter{}
// Version is the human-readable version
Version = "unknown"
// GitCommit hash, set at compile time
GitCommit = "unknown"
)
// infoFormatter overrides the default format for Info() log events to
// provide an easier to read output
type infoFormatter struct {
}
func (f *infoFormatter) Format(entry *log.Entry) ([]byte, error) {
if entry.Level == log.InfoLevel {
return append([]byte(entry.Message), '\n'), nil
}
return defaultLogFormatter.Format(entry)
}
func version() {
fmt.Printf("%s version %s\n", filepath.Base(os.Args[0]), Version)
fmt.Printf("commit: %s\n", GitCommit)
os.Exit(0)
}
func main() {
flag.Usage = func() {
fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0]))
fmt.Printf("Commands:\n")
fmt.Printf(" build Build a Moby image from a YAML file\n")
fmt.Printf(" version Print version information\n")
fmt.Printf(" help Print this message\n")
fmt.Printf("\n")
fmt.Printf("Run '%s COMMAND --help' for more information on the command\n", filepath.Base(os.Args[0]))
fmt.Printf("\n")
fmt.Printf("Options:\n")
flag.PrintDefaults()
}
flagQuiet := flag.Bool("q", false, "Quiet execution")
flagVerbose := flag.Bool("v", false, "Verbose execution")
// Set up logging
log.SetFormatter(new(infoFormatter))
log.SetLevel(log.InfoLevel)
flag.Parse()
if *flagQuiet && *flagVerbose {
fmt.Printf("Can't set quiet and verbose flag at the same time\n")
os.Exit(1)
}
if *flagQuiet {
log.SetLevel(log.ErrorLevel)
}
if *flagVerbose {
// Switch back to the standard formatter
log.SetFormatter(defaultLogFormatter)
log.SetLevel(log.DebugLevel)
}
args := flag.Args()
if len(args) < 1 {
fmt.Printf("Please specify a command.\n\n")
flag.Usage()
os.Exit(1)
}
switch args[0] {
case "build":
build(args[1:])
case "version":
version()
case "help":
flag.Usage()
default:
fmt.Printf("%q is not valid command.\n\n", args[0])
flag.Usage()
os.Exit(1)
}
}

158
src/cmd/moby/output.go
View File

@ -1,158 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"fmt"
"io/ioutil"
"os"
log "github.com/Sirupsen/logrus"
)
const (
bios = "linuxkit/mkimage-iso-bios:6ebdce90f63991eb1d5a578e6570dc1e5781e9fe@sha256:0c6116d4c069d17ebdaa86737841b3be6ae84f6c69a5e79fe59cd8310156aa96"
efi = "linuxkit/mkimage-iso-efi:008fac48c41ec38b36ce1ae62f93a69ee9328569@sha256:35282010b95680fe754e557bc65f0b2ffd85e925bd62f427fb77bf494145083b"
gcp = "linuxkit/mkimage-gcp:a8b909202c0a0ed2ac31b5c21f6701d3253ff29a@sha256:2ba307e537d6fae37115848c8a0f5a9b3ed578e102c93c5d2578ece4a91cb828"
qcow = "linuxkit/mkimage-qcow:a1053b5dc80834adcba2e5f49354f62797e35f84@sha256:3312d523a67e7c7efb3c3eaa5a4dfbd46659549681d6d62cdeb02bd475b3a22c"
vhd = "linuxkit/mkimage-vhd:98d6c879a52cb85b87269bc6ecf9df7dd134427a@sha256:0ca6f46690c7890c77295cc6c531f95fc8bb41df42c237ae4b32eea338cec4e7"
vmdk = "linuxkit/mkimage-vmdk:10b8717b6a2099741b702c31af2d9a42ce50425e@sha256:bf7cf6029e61685e9085a1883b1be1167a7f06199f3b76a944ea87b6f23f60d8"
)
func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
log.Debugf("output: %s %s", m.Outputs, base)
for _, o := range m.Outputs {
switch o.Format {
case "kernel+initrd":
err := outputKernelInitrd(base, bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "iso-bios":
err := outputISO(bios, base+".iso", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "iso-efi":
err := outputISO(efi, base+"-efi.iso", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "gcp-img":
err := outputImg(gcp, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "qcow", "qcow2":
err := outputImg(qcow, base+".qcow2", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "vhd":
err := outputImg(vhd, base+".vhd", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "vmdk":
err := outputImg(vmdk, base+".vmdk", bzimage, initrd, m.Kernel.Cmdline)
if err != nil {
return fmt.Errorf("Error writing %s output: %v", o.Format, err)
}
case "":
return fmt.Errorf("No format specified for output")
default:
return fmt.Errorf("Unknown output type %s", o.Format)
}
}
return nil
}
func tarInitrdKernel(bzimage, initrd []byte) (*bytes.Buffer, error) {
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
hdr := &tar.Header{
Name: "bzImage",
Mode: 0600,
Size: int64(len(bzimage)),
}
err := tw.WriteHeader(hdr)
if err != nil {
return buf, err
}
_, err = tw.Write(bzimage)
if err != nil {
return buf, err
}
hdr = &tar.Header{
Name: "initrd.img",
Mode: 0600,
Size: int64(len(initrd)),
}
err = tw.WriteHeader(hdr)
if err != nil {
return buf, err
}
_, err = tw.Write(initrd)
if err != nil {
return buf, err
}
err = tw.Close()
if err != nil {
return buf, err
}
return buf, nil
}
func outputImg(image, filename string, bzimage []byte, initrd []byte, args ...string) error {
log.Debugf("output img: %s %s", image, filename)
log.Infof(" %s", filename)
buf, err := tarInitrdKernel(bzimage, initrd)
if err != nil {
return err
}
img, err := dockerRunInput(buf, append([]string{image}, args...)...)
if err != nil {
return err
}
err = ioutil.WriteFile(filename, img, os.FileMode(0644))
if err != nil {
return err
}
return nil
}
func outputISO(image, filename string, bzimage []byte, initrd []byte, args ...string) error {
log.Debugf("output iso: %s %s", image, filename)
log.Infof(" %s", filename)
buf, err := tarInitrdKernel(bzimage, initrd)
if err != nil {
return err
}
iso, err := dockerRunInput(buf, append([]string{image}, args...)...)
if err != nil {
return err
}
err = ioutil.WriteFile(filename, iso, os.FileMode(0644))
if err != nil {
return err
}
return nil
}
func outputKernelInitrd(base string, bzimage []byte, initrd []byte, cmdline string) error {
log.Debugf("output kernel/initrd: %s %s", base, cmdline)
log.Infof(" %s %s %s", base+"-bzImage", base+"-initrd.img", base+"-cmdline")
err := ioutil.WriteFile(base+"-initrd.img", initrd, os.FileMode(0644))
if err != nil {
return err
}
err = ioutil.WriteFile(base+"-bzImage", bzimage, os.FileMode(0644))
if err != nil {
return err
}
err = ioutil.WriteFile(base+"-cmdline", []byte(cmdline), os.FileMode(0644))
if err != nil {
return err
}
return nil
}

121
src/cmd/moby/schema.go
View File

@ -1,121 +0,0 @@
package main
var schema = string(`
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Moby Config",
"additionalProperties": false,
"definitions": {
"kernel": {
"type": "object",
"additionalProperties": false,
"properties": {
"image": { "type": "string"},
"cmdline": { "type": "string"}
}
},
"file": {
"type": "object",
"additionalProperties": false,
"properties": {
"path": {"type": "string"},
"directory": {"type": "boolean"},
"contents": {"type": "string"}
}
},
"files": {
"type": "array",
"items": { "$ref": "#/definitions/file" }
},
"output": {
"type": "object",
"additionalProperties": false,
"properties": {
"format": {"type": "string"}
}
},
"outputs": {
"type": "array",
"items": { "$ref": "#/definitions/output" }
},
"trust": {
"type": "object",
"additionalProperties": false,
"properties": {
"image": { "$ref": "#/definitions/strings" },
"org": { "$ref": "#/definitions/strings" }
}
},
"strings": {
"type": "array",
"items": {"type": "string"}
},
"mount": {
"type": "object",
"additionalProperties": false,
"properties": {
"destination": { "type": "string" },
"type": { "type": "string" },
"source": { "type": "string" },
"options": { "$ref": "#/definitions/strings" }
}
},
"mounts": {
"type": "array",
"items": { "$ref": "#/definitions/mount" }
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["name", "image"],
"properties": {
"name": {"type": "string"},
"image": {"type": "string"},
"capabilities": { "$ref": "#/definitions/strings" },
"mounts": { "$ref": "#/definitions/mounts" },
"binds": { "$ref": "#/definitions/strings" },
"tmpfs": { "$ref": "#/definitions/strings" },
"command": { "$ref": "#/definitions/strings" },
"env": { "$ref": "#/definitions/strings" },
"cwd": { "type": "string"},
"net": { "type": "string"},
"pid": { "type": "string"},
"ipc": { "type": "string"},
"uts": { "type": "string"},
"readonly": { "type": "boolean"},
"maskedPaths": { "$ref": "#/definitions/strings" },
"readonlyPaths": { "$ref": "#/definitions/strings" },
"uid": {"type": "integer"},
"gid": {"type": "integer"},
"additionalGids": {
"type": "array",
"items": { "type": "integer" }
},
"noNewPrivileges": {"type": "boolean"},
"hostname": {"type": "string"},
"oomScoreAdj": {"type": "integer"},
"disableOOMKiller": {"type": "boolean"},
"rootfsPropagation": {"type": "string"},
"cgroupsPath": {"type": "string"},
"sysctl": {
"type": "array",
"items": { "$ref": "#/definitions/strings" }
}
}
},
"images": {
"type": "array",
"items": { "$ref": "#/definitions/image" }
}
},
"properties": {
"kernel": { "$ref": "#/definitions/kernel" },
"init": { "$ref": "#/definitions/strings" },
"onboot": { "$ref": "#/definitions/images" },
"services": { "$ref": "#/definitions/images" },
"trust": { "$ref": "#/definitions/trust" },
"files": { "$ref": "#/definitions/files" },
"outputs": { "$ref": "#/definitions/outputs" }
}
}
`)

32
tools/go-compile/compile.sh
View File

@ -30,6 +30,18 @@ do
ldflags="$2"
shift
;;
--clone-path)
clonepath="$2"
shift
;;
--clone)
clone="$2"
shift
;;
--commit)
commit="$2"
shift
;;
*)
echo "Unknown option $1"
exit 1
@ -44,12 +56,22 @@ done
dir="$GOPATH/src/$package"
mkdir -p $dir
if [ -z "$clone" ]
then
mkdir -p "$dir"
cd "$dir"
# untar input
tar xf -
else
mkdir -p "$GOPATH/src/$clonepath"
cd "$GOPATH/src/$clonepath"
git clone "$clone" .
[ ! -z "$commit" ] && git checkout "$commit"
mkdir -p "$dir"
cd "$dir"
fi
# untar input
tar xf - -C $dir
cd $dir
cd "$dir"
# lint before building
>&2 echo "gofmt..."