Update moby tool and qemu fixes

- no longer uses several of the `mkimage-*` tools in favour of dogfooding
with `linuxkit` and using the `mkimage` package.
- fix the qemu docker container fallbacks to work better when multiple
paths are used for disks and the image.

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

Makefile

@@ -17,7 +17,7 @@ endif
 
 PREFIX?=/usr/local/
 
-MOBY_COMMIT=cde8329d2f72ef22b8ee7e817022770c33b521df
+MOBY_COMMIT=3a16c02f100d1e01ea405a33d21a02de2a85904f
 bin/moby: Makefile | bin
 	docker run --rm --log-driver=none $(CROSS) $(GO_COMPILE) --clone-path github.com/moby/tool --clone https://github.com/moby/tool.git --commit $(MOBY_COMMIT) --package github.com/moby/tool/cmd/moby --ldflags "-X main.GitCommit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o $@ > tmp_moby_bin.tar
 	tar xf tmp_moby_bin.tar > $@

src/cmd/linuxkit/run_qemu.go

@@ -208,24 +208,32 @@ func runQemuLocal(config QemuConfig) error {
 }
 
 func runQemuContainer(config QemuConfig) error {
-	var wd string
-	if filepath.IsAbs(config.Path) {
-		// Split the path
-		wd, config.Path = filepath.Split(config.Path)
-		log.Debugf("Path: %s", config.Path)
-	} else {
-		var err error
-		wd, err = os.Getwd()
+	cwd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
+
+	var binds []string
+	if filepath.IsAbs(config.Path) {
+		binds = append(binds, "-v", fmt.Sprintf("%[1]s:%[1]s", filepath.Dir(config.Path)))
+	} else {
+		binds = append(binds, "-v", fmt.Sprintf("%[1]s:%[1]s", cwd))
+	}
+
+	// also try to bind mount disk paths so the command works
+	for _, d := range config.Disks {
+		if filepath.IsAbs(d.Path) {
+			binds = append(binds, "-v", fmt.Sprintf("%[1]s:%[1]s", filepath.Dir(d.Path)))
+		} else {
+			binds = append(binds, "-v", fmt.Sprintf("%[1]s:%[1]s", cwd))
+		}
 	}
 
 	var args []string
 	config, args = buildQemuCmdline(config)
 
-	dockerArgs := []string{"run", "--interactive", "--rm", "-v", fmt.Sprintf("%s:%s", wd, "/tmp"), "-w", "/tmp"}
-	dockerArgsImg := []string{"run", "--rm", "-v", fmt.Sprintf("%s:%s", wd, "/tmp"), "-w", "/tmp"}
+	dockerArgs := append([]string{"run", "--interactive", "--rm", "-w", cwd}, binds...)
+	dockerArgsImg := append([]string{"run", "--rm", "-w", cwd}, binds...)
 
 	if terminal.IsTerminal(int(os.Stdin.Fd())) {
 		dockerArgs = append(dockerArgs, "--tty")
@@ -327,8 +335,8 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
 
 	// build kernel boot config from kernel/initrd/cmdline
 	if config.Kernel {
-		qemuKernelPath := buildPath(config.Path, "-kernel")
-		qemuInitrdPath := buildPath(config.Path, "-initrd.img")
+		qemuKernelPath := config.Path + "-kernel"
+		qemuInitrdPath := config.Path + "-initrd.img"
 		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
 		qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath)
 		cmdlineString, err := ioutil.ReadFile(config.Path + "-cmdline")
@@ -377,16 +385,6 @@ func discoverBackend(config QemuConfig) QemuConfig {
 	return config
 }
 
-func buildPath(prefix string, postfix string) string {
-	path := prefix + postfix
-	if filepath.IsAbs(path) {
-		if _, err := os.Stat(path); os.IsNotExist(err) {
-			log.Fatalf("File [%s] does not exist in current directory", path)
-		}
-	}
-	return path
-}
-
 type multipleFlag []string
 
 type publishedPorts struct {

test/cases/000_build/000_outputs/test.sh

@@ -17,13 +17,14 @@ clean_up() {
 
 trap clean_up EXIT
 
-moby build -output tar,kernel+initrd,iso-bios,iso-efi,img-gz,qcow2,vmdk -name "${NAME}" test.yml
+moby build -disable-content-trust -output tar,kernel+initrd,iso-bios,iso-efi,img,img-gz,qcow2,vmdk -name "${NAME}" test.yml
 [ -f "${NAME}.tar" ] || exit 1
 [ -f "${NAME}-kernel" ] || exit 1
 [ -f "${NAME}-initrd.img" ] || exit 1
 [ -f "${NAME}-cmdline" ] || exit 1
 [ -f "${NAME}.iso" ] || exit 1
 [ -f "${NAME}-efi.iso" ] || exit 1
+[ -f "${NAME}.img" ] || exit 1
 [ -f "${NAME}.img.gz" ] || exit 1
 [ -f "${NAME}.qcow2" ] || exit 1
 # VHD currently requires a lot of memory, disable for now

test/cases/010_platforms/000_qemu/040_run_raw/test.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# SUMMARY: Check that raw image boots in qemu
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+NAME=qemu-raw
+
+clean_up() {
+	# remove any files, containers, images etc
+	rm -rf ${NAME}* || true
+}
+
+trap clean_up EXIT
+
+moby build -output img -name "${NAME}" test.yml
+[ -f "${NAME}.img" ] || exit 1
+linuxkit run qemu "${NAME}.img" | grep -q "Welcome to LinuxKit"
+exit 0

test/cases/010_platforms/000_qemu/040_run_raw/test.yml

@@ -0,0 +1,14 @@
+kernel:
+  image: "linuxkit/kernel:4.9.x"
+  cmdline: "console=ttyS0 console=tty0 page_poison=1"
+init:
+  - linuxkit/init:1b8a7e394d2ec2f1fdb4d67645829d1b5bdca037
+  - linuxkit/runc:3a4e6cbf15470f62501b019b55e1caac5ee7689f
+  - linuxkit/containerd:deaf5bf838bf7f131c2287ecff3ed9835b0497e2
+onboot:
+  - name: poweroff
+    image: "linuxkit/poweroff:7404cf2295df89ccfa2dda41997a28307a90cf28"
+    command: ["/bin/sh", "/poweroff.sh", "10"]
+trust:
+  org:
+    - linuxkit

tools/mkimage-gcp/Dockerfile

@@ -1,5 +0,0 @@
-FROM linuxkit/guestfs:2657580764d5791e103647237dac5a0276533e2e@sha256:8f99eba6720df17bce8893052d7ca9a07cdc9cd09b335a5a9c57ebd5a44934be
-
-COPY . .
-
-ENTRYPOINT [ "/make-gcp" ]

tools/mkimage-gcp/Makefile

@@ -1,27 +0,0 @@
-.PHONY: tag push
-
-IMAGE=mkimage-gcp
-
-default: push
-
-hash: Dockerfile make-gcp
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --entrypoint sh --rm $(IMAGE):build -c "(cat $^; apt list --installed 2>/dev/null) | sha1sum" | sed 's/ .*//' > hash
-
-push: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		(docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash) && \
-		 docker push linuxkit/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-tag: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:

tools/mkimage-gcp/make-gcp

@@ -1,55 +0,0 @@
-#!/bin/sh
-
-set -e
-
-mkdir -p /tmp/image
-cd /tmp/image
-
-# input is a tarball of kernel and initrd.img on stdin
-# output is a compressed tarball of a raw disk image on stdout
-
-mkdir -p files
-
-cd files
-
-# extract. As guestfs base is currently Debian, no compression support
-# only if stdin is a tty, if so need files volume mounted...
-[ -t 0 ] || tar xf -
-
-INITRD="$(find . -name '*.img')"
-KERNEL="$(find . -name kernel -or -name bzImage)"
-CMDLINE="$*"
-
-[ "$KERNEL" = "./kernel" ] || mv "$KERNEL" kernel
-[ "$INITRD" = "./initrd.img" ] || mv "$INITRD" initrd.img
-
-# clean up subdirectories
-find . -mindepth 1 -maxdepth 1 -type d | xargs rm -rf
-
-# should be externally provided as GCP specific
-GCP_CONFIG="earlyprintk=ttyS0,115200 console=ttyS0,115200 vsyscall=emulate page_poison=1"
-
-CFG="DEFAULT linux
-LABEL linux
-    KERNEL /kernel
-    INITRD /initrd.img
-    APPEND ${CMDLINE}
-"
-
-printf "$CFG" > syslinux.cfg
-
-cd ..
-
-tar cf files.tar -C files .
-
-virt-make-fs --size=1G --type=ext4 --partition files.tar disk.raw
-
-guestfish -a disk.raw -m /dev/sda1 <<EOF
-  upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin
-  copy-file-to-device /mbr.bin /dev/sda size:440
-  rm /mbr.bin
-  extlinux /
-  part-set-bootable /dev/sda 1 true
-EOF
-
-tar cf - disk.raw | gzip -9

tools/mkimage-img-gz/Dockerfile

@@ -1,5 +0,0 @@
-FROM linuxkit/guestfs:2657580764d5791e103647237dac5a0276533e2e@sha256:8f99eba6720df17bce8893052d7ca9a07cdc9cd09b335a5a9c57ebd5a44934be
-
-COPY . .
-
-ENTRYPOINT [ "/make-img-gz" ]

tools/mkimage-img-gz/Makefile

@@ -1,27 +0,0 @@
-.PHONY: tag push
-
-IMAGE=mkimage-img-gz
-
-default: push
-
-hash: Dockerfile make-img-gz
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --entrypoint sh --rm $(IMAGE):build -c "(cat $^; apt list --installed 2>/dev/null) | sha1sum" | sed 's/ .*//' > hash
-
-push: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		(docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash) && \
-		 docker push linuxkit/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-tag: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:

tools/mkimage-img-gz/make-img-gz

@@ -1,56 +0,0 @@
-#!/bin/sh
-
-set -e
-
-SIZE="$1"
-[ -z "$SIZE" ] && SIZE=1G
-
-mkdir -p /tmp/image
-cd /tmp/image
-
-# input is a tarball of kernel and initrd.img on stdin
-# output is a compressed raw disk image on stdout
-
-mkdir -p files
-
-cd files
-
-# extract. As guestfs base is currently Debian, no compression support
-# only if stdin is a tty, if so need files volume mounted...
-[ -t 0 ] || tar xf -
-
-INITRD="$(find . -name '*.img')"
-KERNEL="$(find . -name kernel -or -name bzImage)"
-CMDLINE_FILE="$(find . -name cmdline)"
-CMDLINE="$(cat $CMDLINE_FILE)"
-
-[ "$KERNEL" = "./kernel" ] || mv "$KERNEL" kernel
-[ "$INITRD" = "./initrd.img" ] || mv "$INITRD" initrd.img
-
-# clean up subdirectories
-find . -mindepth 1 -maxdepth 1 -type d | xargs rm -rf
-
-CFG="DEFAULT linux
-LABEL linux
-    KERNEL /kernel
-    INITRD /initrd.img
-    APPEND ${CMDLINE}
-"
-
-printf "$CFG" > syslinux.cfg
-
-cd ..
-
-tar cf files.tar -C files .
-
-virt-make-fs --size=${SIZE} --type=ext4 --partition files.tar disk.raw
-
-guestfish -a disk.raw -m /dev/sda1 <<EOF
-  upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin
-  copy-file-to-device /mbr.bin /dev/sda size:440
-  rm /mbr.bin
-  extlinux /
-  part-set-bootable /dev/sda 1 true
-EOF
-
-cat disk.raw | gzip -9

tools/mkimage-qcow/Dockerfile

@@ -1,5 +0,0 @@
-FROM linuxkit/guestfs:2657580764d5791e103647237dac5a0276533e2e@sha256:8f99eba6720df17bce8893052d7ca9a07cdc9cd09b335a5a9c57ebd5a44934be
-
-COPY . .
-
-ENTRYPOINT [ "/make-qcow" ]

tools/mkimage-qcow/Makefile

@@ -1,27 +0,0 @@
-.PHONY: tag push
-
-IMAGE=mkimage-qcow
-
-default: push
-
-hash: Dockerfile make-qcow
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --entrypoint sh --rm $(IMAGE):build -c "(cat $^; apt list --installed 2>/dev/null) | sha1sum" | sed 's/ .*//' > hash
-
-push: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		(docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash) && \
-		 docker push linuxkit/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-tag: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:

tools/mkimage-qcow/make-qcow

@@ -1,55 +0,0 @@
-#!/bin/sh
-
-set -e
-
-mkdir -p /tmp/image
-cd /tmp/image
-
-# input is a tarball of kernel and initrd.img on stdin
-# output is a qcow2 on stdout
-
-mkdir -p files
-
-cd files
-
-# extract. As guestfs base is currently Debian, no compression support
-# only if stdin is a tty, if so need files volume mounted...
-[ -t 0 ] || tar xf -
-
-INITRD="$(find . -name '*.img')"
-KERNEL="$(find . -name kernel -or -name '*bzImage')"
-CMDLINE="$*"
-
-[ "$KERNEL" = "./kernel" ] || mv "$KERNEL" kernel
-[ "$INITRD" = "./initrd.img" ] || mv "$INITRD" initrd.img
-
-# clean up subdirectories
-find . -mindepth 1 -maxdepth 1 -type d | xargs rm -rf
-
-CFG="DEFAULT linux
-LABEL linux
-    KERNEL /kernel
-    INITRD /initrd.img
-    APPEND ${CMDLINE}
-"
-
-printf "$CFG" > syslinux.cfg
-
-cd ..
-
-tar cf files.tar -C files .
-
-virt-make-fs --size=1G --type=ext4 --partition files.tar --format=qcow2 disk.qcow
-
-guestfish -a disk.qcow -m /dev/sda1 <<EOF
-  upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin
-  copy-file-to-device /mbr.bin /dev/sda size:440
-  rm /mbr.bin
-  extlinux /
-  part-set-bootable /dev/sda 1 true
-EOF
-
-# compress qcow
-qemu-img convert -f qcow2 -O qcow2 -c disk.qcow disk.qcow2 1>&2
-
-cat disk.qcow2