Merge pull request #1379 from justincormack/pull

Add a moby build --pull option to force pulling images

src/cmd/moby/build.go

@@ -65,7 +65,7 @@ func containersInitrd(containers []*bytes.Buffer) (*bytes.Buffer, error) {
 	return w, nil
 }
 
-func build(name string, args []string) {
+func build(name string, pull bool, args []string) {
 	conf := "moby.yaml"
 	if len(args) > 0 {
 		conf = args[0]
@@ -91,6 +91,12 @@ func build(name string, args []string) {
 
 	containers := []*bytes.Buffer{}
 
+	if pull {
+		err := dockerPull(m.Kernel.Image)
+		if err != nil {
+			log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
+		}
+	}
 	// get kernel bzImage and initrd tarball from container
 	// TODO examine contents to see what names they might have
 	const (
@@ -109,6 +115,12 @@ func build(name string, args []string) {
 	containers = append(containers, ktar)
 
 	// convert init image to tarball
+	if pull {
+		err := dockerPull(m.Init)
+		if err != nil {
+			log.Fatalf("Could not pull image %s: %v", m.Init, err)
+		}
+	}
 	init, err := ImageExtract(m.Init, "")
 	if err != nil {
 		log.Fatalf("Failed to build init tarball: %v", err)
@@ -117,6 +129,12 @@ func build(name string, args []string) {
 	containers = append(containers, buffer)
 
 	for i, image := range m.System {
+		if pull {
+			err := dockerPull(image.Image)
+			if err != nil {
+				log.Fatalf("Could not pull image %s: %v", image.Image, err)
+			}
+		}
 		config, err := ConfigToOCI(&image)
 		if err != nil {
 			log.Fatalf("Failed to run riddler to get config.json for %s: %v", image.Image, err)
@@ -132,6 +150,12 @@ func build(name string, args []string) {
 	}
 
 	for _, image := range m.Daemon {
+		if pull {
+			err := dockerPull(image.Image)
+			if err != nil {
+				log.Fatalf("Could not pull image %s: %v", image.Image, err)
+			}
+		}
 		config, err := ConfigToOCI(&image)
 		if err != nil {
 			log.Fatalf("Failed to run riddler to get config.json for %s: %v", image.Image, err)

src/cmd/moby/docker.go

@@ -0,0 +1,262 @@
+package main
+
+// We want to replace much of this with use of containerd tools
+// and also using the Docker API not shelling out
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+)
+
+func dockerRun(args ...string) ([]byte, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args = append([]string{"run", "--rm"}, args...)
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%v: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func dockerRunInput(input io.Reader, args ...string) ([]byte, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args = append([]string{"run", "--rm", "-i"}, args...)
+	cmd := exec.Command(docker, args...)
+	cmd.Stdin = input
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%v: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func dockerCreate(image string) (string, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return "", errors.New("Docker does not seem to be installed")
+	}
+	// we do not ever run the container, so /dev/null is used as command
+	args := []string{"create", image, "/dev/null"}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return "", err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return "", err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return "", err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return "", err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	container := strings.TrimSpace(string(stdout))
+	return container, nil
+}
+
+func dockerExport(container string) ([]byte, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args := []string{"export", container}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%v: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func dockerRm(container string) error {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return errors.New("Docker does not seem to be installed")
+	}
+	args := []string{"rm", container}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	_, err = ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return nil
+}
+
+func dockerPull(image string) error {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return errors.New("Docker does not seem to be installed")
+	}
+	args := []string{"pull", image}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	_, err = ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return nil
+}

src/cmd/moby/image.go

@@ -3,11 +3,9 @@ package main
 import (
 	"archive/tar"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os/exec"
 	"strings"
 )
 
@@ -15,131 +13,6 @@ import (
 // used the containerd libraries to do this instead locally direct from a local image
 // cache as it would be much simpler.
 
-func dockerCreate(image string) (string, error) {
-	docker, err := exec.LookPath("docker")
-	if err != nil {
-		return "", errors.New("Docker does not seem to be installed")
-	}
-	// we do not ever run the container, so /dev/null is used as command
-	args := []string{"create", image, "/dev/null"}
-	cmd := exec.Command(docker, args...)
-
-	stderrPipe, err := cmd.StderrPipe()
-	if err != nil {
-		return "", err
-	}
-
-	stdoutPipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return "", err
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		return "", err
-	}
-
-	stdout, err := ioutil.ReadAll(stdoutPipe)
-	if err != nil {
-		return "", err
-	}
-
-	stderr, err := ioutil.ReadAll(stderrPipe)
-	if err != nil {
-		return "", err
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		return "", fmt.Errorf("%s: %s", err, stderr)
-	}
-
-	container := strings.TrimSpace(string(stdout))
-	return container, nil
-}
-
-func dockerExport(container string) ([]byte, error) {
-	docker, err := exec.LookPath("docker")
-	if err != nil {
-		return []byte{}, errors.New("Docker does not seem to be installed")
-	}
-	args := []string{"export", container}
-	cmd := exec.Command(docker, args...)
-
-	stderrPipe, err := cmd.StderrPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdoutPipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdout, err := ioutil.ReadAll(stdoutPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stderr, err := ioutil.ReadAll(stderrPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
-	}
-
-	return stdout, nil
-}
-
-func dockerRm(container string) error {
-	docker, err := exec.LookPath("docker")
-	if err != nil {
-		return errors.New("Docker does not seem to be installed")
-	}
-	args := []string{"rm", container}
-	cmd := exec.Command(docker, args...)
-
-	stderrPipe, err := cmd.StderrPipe()
-	if err != nil {
-		return err
-	}
-
-	stdoutPipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		return err
-	}
-
-	_, err = ioutil.ReadAll(stdoutPipe)
-	if err != nil {
-		return err
-	}
-
-	stderr, err := ioutil.ReadAll(stderrPipe)
-	if err != nil {
-		return err
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		return fmt.Errorf("%s: %s", err, stderr)
-	}
-
-	return nil
-}
-
 var exclude = map[string]bool{
 	".dockerenv":  true,
 	"Dockerfile":  true,

src/cmd/moby/main.go

@@ -1,100 +1,11 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
-	"os/exec"
 )
 
-func dockerRun(args ...string) ([]byte, error) {
-	// TODO switch to using Docker client API not exec - just a quick prototype
-	docker, err := exec.LookPath("docker")
-	if err != nil {
-		return []byte{}, errors.New("Docker does not seem to be installed")
-	}
-	args = append([]string{"run", "--rm"}, args...)
-	cmd := exec.Command(docker, args...)
-
-	stderrPipe, err := cmd.StderrPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdoutPipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdout, err := ioutil.ReadAll(stdoutPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stderr, err := ioutil.ReadAll(stderrPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
-	}
-
-	return stdout, nil
-}
-
-func dockerRunInput(input io.Reader, args ...string) ([]byte, error) {
-	// TODO switch to using Docker client API not exec - just a quick prototype
-	docker, err := exec.LookPath("docker")
-	if err != nil {
-		return []byte{}, errors.New("Docker does not seem to be installed")
-	}
-	args = append([]string{"run", "--rm", "-i"}, args...)
-	cmd := exec.Command(docker, args...)
-	cmd.Stdin = input
-
-	stderrPipe, err := cmd.StderrPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdoutPipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stdout, err := ioutil.ReadAll(stdoutPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	stderr, err := ioutil.ReadAll(stderrPipe)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
-	}
-
-	return stdout, nil
-}
-
 func main() {
 	flag.Usage = func() {
 		fmt.Printf("USAGE: %s COMMAND\n\n", os.Args[0])
@@ -114,6 +25,7 @@ func main() {
 		buildCmd.PrintDefaults()
 	}
 	buildName := buildCmd.String("name", "", "Name to use for output files")
+	buildPull := buildCmd.Bool("pull", false, "Always pull images")
 
 	runCmd := flag.NewFlagSet("run", flag.ExitOnError)
 	runCmd.Usage = func() {
@@ -141,7 +53,7 @@ func main() {
 	switch os.Args[1] {
 	case "build":
 		buildCmd.Parse(os.Args[2:])
-		build(*buildName, buildCmd.Args())
+		build(*buildName, *buildPull, buildCmd.Args())
 	case "run":
 		runCmd.Parse(os.Args[2:])
 		run(*runCPUs, *runMem, *runDiskSz, *runDisk, runCmd.Args())