Merge pull request #1672 from dave-tucker/qemu-container

Let moby run qemu fallback to using a container

Makefile

@@ -42,7 +42,7 @@ test-bzImage: test-initrd.img
 # interactive versions need to use volume mounts
 .PHONY: test-qemu-efi
 test-qemu-efi: test-efi.iso
-	./scripts/qemu.sh $^ 2>&1 | tee test-efi.log
+	$(MOBY) run $^ | tee test-efi.log
 	$(call check_test_log, test-efi.log)
 
 bin:
@@ -58,17 +58,17 @@ endef
 .PHONY: test-hyperkit
 test-hyperkit: $(MOBY) test-initrd.img test-bzImage test-cmdline
 	rm -f disk.img
-	script -q /dev/null $(MOBY) run test | tee test.log
+	$(MOBY) run test | tee test.log
 	$(call check_test_log, test.log)
 
 .PHONY: test-gcp
 test-gcp: $(MOBY) test.img.tar.gz
-	script -q /dev/null $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log
+	$(MOBY) run gcp test.img.tar.gz | tee test-gcp.log
 	$(call check_test_log, test-gcp.log)
 
 .PHONY: test
 test: test-initrd.img test-bzImage test-cmdline
-	tar cf - $^ | ./scripts/qemu.sh 2>&1 | tee test.log
+	$(MOBY) run test | tee test.log
 	$(call check_test_log, test.log)
 
 test-ltp.img.tar.gz: $(MOBY) test/ltp/test-ltp.yml

scripts/qemu.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-
-QEMU_IMAGE=linuxkit/qemu:4563d58e97958f4941fbef9e74cabc08bd402144@sha256:b2db0b13ba1cbb6b48218f088fe0a4d860e1db2c4c6381b5416536f48a612230
-
-# if not interactive
-if [ ! -t 0 -a -z "$1" ]
-then
-	# non interactive, tarball input
-	docker run -i --rm "$QEMU_IMAGE"
-	exit $?
-fi
-
-FILE=$1
-FILE2=$2
-CMDLINE=$3
-[ -z "$FILE" ] && FILE="$PWD/moby"
-
-BASE=$(basename "$FILE")
-DIR=$(dirname "$FILE")
-if [ ! -f "$FILE" -a -f $DIR/$BASE-initrd.img -a -f $DIR/$BASE-bzImage ]
-then
-	FILE=$DIR/$BASE-initrd.img
-	FILE2=$DIR/$BASE-bzImage
-fi
-
-echo "$FILE" | grep -q '^/' || FILE="$PWD/$FILE"
-if [ ! -z "$FILE2" ]
-then
-	echo "$FILE2" | grep -q '^/' || FILE2="$PWD/$FILE2"
-fi
-if [ ! -z "$CMDLINE" ]
-then
-	echo "$CMDLINE" | grep -q '^/' || CMDLINE="$PWD/$CMDLINE"
-fi
-
-if [ -c "/dev/kvm" ] ; then
-    DEVKVM="--device=/dev/kvm"
-fi
-BASE=$(basename "$FILE")
-MOUNTS="-v $FILE:/tmp/$BASE"
-BASE2=$(basename "$FILE2")
-BASE3=$(basename "$CMDLINE")
-
-[ ! -z "$FILE2" ] && MOUNTS="$MOUNTS -v $FILE2:/tmp/$BASE2"
-[ ! -z "$CMDLINE" ] && MOUNTS="$MOUNTS -v $CMDLINE:/tmp/$BASE3"
-
-docker run -it --rm $MOUNTS $DEVKVM "$QEMU_IMAGE"

src/cmd/moby/run_qemu.go

@@ -10,6 +10,28 @@ import (
 	log "github.com/Sirupsen/logrus"
 )
 
+// QemuImg is the version of qemu container
+const QemuImg = "linuxkit/qemu:17f052263d63c8a2b641ad91c589edcbb8a18c82"
+
+// QemuConfig contains the config for Qemu
+type QemuConfig struct {
+	Prefix        string
+	ISO           bool
+	UEFI          bool
+	Kernel        bool
+	GUI           bool
+	DiskPath      string
+	DiskSize      string
+	FWPath        string
+	Arch          string
+	CPUs          string
+	Memory        string
+	KVM           bool
+	Containerized bool
+	QemuBinPath   string
+	QemuImgPath   string
+}
+
 func runQemu(args []string) {
 	qemuFlags := flag.NewFlagSet("qemu", flag.ExitOnError)
 	qemuFlags.Usage = func() {
@@ -53,78 +75,192 @@ func runQemu(args []string) {
 		log.Warnf("Both -iso and -uefi have been used")
 	}
 
-	// Before building qemu arguments, check qemu is in the $PATH
+	config := QemuConfig{
-	qemuBinPath := "qemu-system-" + *qemuArch
+		Prefix:   prefix,
-	qemuImgPath := "qemu-img"
+		ISO:      *qemuIso,
-	fullQemuPath, err := exec.LookPath(qemuBinPath)
+		UEFI:     *qemuUEFI,
+		Kernel:   *qemuKernel,
+		GUI:      *qemuGUI,
+		DiskPath: *qemuDiskPath,
+		DiskSize: *qemuDiskSize,
+		FWPath:   *qemuFWPath,
+		Arch:     *qemuArch,
+		CPUs:     *qemuCPUs,
+		Memory:   *qemuMem,
+	}
+
+	config, qemuArgs := buildQemuCmdline(config)
+
+	var err error
+	if config.Containerized {
+		err = runQemuContainer(config, qemuArgs)
+	} else {
+		err = runQemuLocal(config, qemuArgs)
+	}
 	if err != nil {
-		log.Fatalf("Unable to find %s within the $PATH", qemuBinPath)
+		log.Fatal(err.Error())
+	}
+}
+
+func runQemuLocal(config QemuConfig, args []string) error {
+	if config.DiskPath != "" {
+		// If disk doesn't exist then create one
+		if _, err := os.Stat(config.DiskPath); err != nil {
+			if os.IsNotExist(err) {
+				log.Infof("Creating new qemu disk [%s]", config.DiskPath)
+				qemuImgCmd := exec.Command(config.QemuImgPath, "create", "-f", "qcow2", config.DiskPath, config.DiskSize)
+				log.Debugf("%v\n", qemuImgCmd.Args)
+				if err := qemuImgCmd.Run(); err != nil {
+					return fmt.Errorf("Error creating disk [%s]:  %s", config.DiskPath, err.Error())
+				}
+			} else {
+				return err
+			}
+		} else {
+			log.Infof("Using existing disk [%s]", config.DiskPath)
+		}
+	}
+
+	// Check for OVMF firmware before running
+	if config.UEFI {
+		if _, err := os.Stat(config.FWPath); err != nil {
+			if os.IsNotExist(err) {
+				return fmt.Errorf("File [%s] does not exist, please ensure OVMF is installed", config.FWPath)
+			}
+			return err
+		}
+	}
+
+	qemuCmd := exec.Command(config.QemuBinPath, args...)
+	// If verbosity is enabled print out the full path/arguments
+	log.Debugf("%v\n", qemuCmd.Args)
+
+	// If we're not using a separate window then link the execution to stdin/out
+	if config.GUI != true {
+		qemuCmd.Stdin = os.Stdin
+		qemuCmd.Stdout = os.Stdout
+		qemuCmd.Stderr = os.Stderr
+	}
+
+	return qemuCmd.Run()
+}
+
+func runQemuContainer(config QemuConfig, args []string) error {
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	dockerArgs := []string{"run", "-i", "--rm", "-v", fmt.Sprintf("%s:%s", wd, "/tmp"), "-w", "/tmp"}
+
+	if config.KVM {
+		dockerArgs = append(dockerArgs, "--device", "/dev/kvm")
+	}
+
+	dockerPath, err := exec.LookPath("docker")
+	if err != nil {
+		return fmt.Errorf("Unable to find docker in the $PATH")
+	}
+
+	if config.DiskPath != "" {
+		// If disk doesn't exist then create one
+		if _, err = os.Stat(config.DiskPath); err != nil {
+			if os.IsNotExist(err) {
+				log.Infof("Creating new qemu disk [%s]", config.DiskPath)
+				imgArgs := append(dockerArgs, QemuImg, "qemu-img", "create", "-f", "qcow2", config.DiskPath, config.DiskSize)
+				qemuImgCmd := exec.Command(dockerPath, imgArgs...)
+				log.Debugf("%v\n", qemuImgCmd.Args)
+				if err = qemuImgCmd.Run(); err != nil {
+					return fmt.Errorf("Error creating disk [%s]:  %s", config.DiskPath, err.Error())
+				}
+			} else {
+				return err
+			}
+		} else {
+			log.Infof("Using existing disk [%s]", config.DiskPath)
+		}
+	}
+
+	qemuArgs := append(dockerArgs, QemuImg, "qemu-system-"+config.Arch)
+	qemuArgs = append(qemuArgs, args...)
+	qemuCmd := exec.Command(dockerPath, qemuArgs...)
+	// If verbosity is enabled print out the full path/arguments
+	log.Debugf("%v\n", qemuCmd.Args)
+
+	// GUI mode not currently supported in a container. Although it could be in future.
+	if config.GUI == true {
+		return fmt.Errorf("GUI mode is only supported when running locally, not in a container")
+	}
+
+	qemuCmd.Stdin = os.Stdin
+	qemuCmd.Stdout = os.Stdout
+	qemuCmd.Stderr = os.Stderr
+
+	return qemuCmd.Run()
+}
+
+func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
+	// Before building qemu arguments, check if qemu is in the PATH or fallback to containerized
+	qemuBinPath := "qemu-system-" + config.Arch
+	qemuImgPath := "qemu-img"
+
+	var err error
+	config.QemuBinPath, err = exec.LookPath(qemuBinPath)
+	if err != nil {
+		log.Infof("Unable to find %s within the $PATH. Using a container", qemuBinPath)
+		config.Containerized = true
+	}
+
+	config.QemuImgPath, err = exec.LookPath(qemuImgPath)
+	if err != nil {
+		// No need to show the error message twice
+		if !config.Containerized {
+			log.Infof("Unable to find %s within the $PATH. Using a container", qemuImgPath)
+			config.Containerized = true
+		}
 	}
 
 	// Iterate through the flags and build arguments
 	var qemuArgs []string
 	qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci")
-	qemuArgs = append(qemuArgs, "-smp", *qemuCPUs)
+	qemuArgs = append(qemuArgs, "-smp", config.CPUs)
-	qemuArgs = append(qemuArgs, "-m", *qemuMem)
+	qemuArgs = append(qemuArgs, "-m", config.Memory)
 
 	// Look for kvm device and enable for qemu if it exists
 	if _, err = os.Stat("/dev/kvm"); os.IsNotExist(err) {
 		qemuArgs = append(qemuArgs, "-machine", "q35")
 	} else {
+		config.KVM = true
 		qemuArgs = append(qemuArgs, "-enable-kvm")
 		qemuArgs = append(qemuArgs, "-machine", "q35,accel=kvm:tcg")
 	}
 
-	if *qemuDiskPath != "" {
+	if config.DiskPath != "" {
-		// If disk doesn't exist then create one
+		qemuArgs = append(qemuArgs, "-drive", "file="+config.DiskPath+",format=qcow2")
-		if _, err = os.Stat(*qemuDiskPath); os.IsNotExist(err) {
-			log.Infof("Creating new qemu disk [%s]", *qemuDiskPath)
-			fullQemuImgPath, err := exec.LookPath(qemuImgPath)
-			if err != nil {
-				log.Fatalf("Unable to find %s within the $PATH", qemuImgPath)
-			}
-			cmd := exec.Command(fullQemuImgPath, "create", "-f", "qcow2", *qemuDiskPath, *qemuDiskSize)
-			if err = cmd.Run(); err != nil {
-				log.Fatalf("Error creating disk [%s]:  %s", *qemuDiskPath, err.Error())
-			}
-		} else {
-			log.Infof("Using existing disk [%s]", *qemuDiskPath)
-		}
-		qemuArgs = append(qemuArgs, "-drive", "file="+*qemuDiskPath+",format=qcow2")
 	}
 
 	// Check flags for iso/uefi boot and if so disable kernel boot
-	if *qemuIso {
+	if config.ISO {
-		*qemuKernel = false
+		config.Kernel = false
-		qemuIsoPath := buildPath(prefix, ".iso")
+		qemuIsoPath := buildPath(config.Prefix, ".iso")
 		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
 	}
 
-	if *qemuUEFI {
+	if config.UEFI {
-		// Check for OVMF firmware before building paths
+		config.Kernel = false
-		_, err = os.Stat(*qemuFWPath)
+		qemuIsoPath := buildPath(config.Prefix, "-efi.iso")
-		if err != nil {
+		qemuArgs = append(qemuArgs, "-pflash", config.FWPath)
-			if os.IsNotExist(err) {
-				log.Fatalf("File [%s] does not exist, please ensure OVMF is installed", *qemuFWPath)
-			} else {
-				log.Fatalf("%s", err.Error())
-			}
-		}
-
-		*qemuKernel = false
-		qemuIsoPath := buildPath(prefix, "-efi.iso")
-		qemuArgs = append(qemuArgs, "-pflash", *qemuFWPath)
 		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
 		qemuArgs = append(qemuArgs, "-boot", "d")
 	}
 
 	// build kernel boot config from bzImage/initrd/cmdline
-	if *qemuKernel {
+	if config.Kernel {
-		qemuKernelPath := buildPath(prefix, "-bzImage")
+		qemuKernelPath := buildPath(config.Prefix, "-bzImage")
-		qemuInitrdPath := buildPath(prefix, "-initrd.img")
+		qemuInitrdPath := buildPath(config.Prefix, "-initrd.img")
 		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
 		qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath)
-		consoleString, err := ioutil.ReadFile(prefix + "-cmdline")
+		consoleString, err := ioutil.ReadFile(config.Prefix + "-cmdline")
 		if err != nil {
 			log.Infof(" %s\n defaulting to console output", err.Error())
 			qemuArgs = append(qemuArgs, "-append", "console=ttyS0 console=tty0 page_poison=1")
@@ -133,25 +269,11 @@ func runQemu(args []string) {
 		}
 	}
 
-	if *qemuGUI != true {
+	if config.GUI != true {
 		qemuArgs = append(qemuArgs, "-nographic")
 	}
 
-	// If verbosity is enabled print out the full path/arguments
+	return config, qemuArgs
-	log.Debugf("%s %v\n", fullQemuPath, qemuArgs)
-
-	cmd := exec.Command(fullQemuPath, qemuArgs...)
-
-	// If we're not using a seperate window then link the execution to stdin/out
-	if *qemuGUI != true {
-		cmd.Stdin = os.Stdin
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-	}
-
-	if err = cmd.Run(); err != nil {
-		log.Fatalf("Error starting %s: %s", fullQemuPath, err.Error())
-	}
 }
 
 func buildPath(prefix string, postfix string) string {

tools/qemu/Dockerfile

@@ -13,4 +13,3 @@ RUN \
   && true
 
 COPY . .
-ENTRYPOINT ["/qemu.sh"]

tools/qemu/Makefile

@@ -5,10 +5,10 @@ IMAGE=qemu
 
 default: push
 
-hash: Dockerfile qemu.sh repositories
+hash: Dockerfile repositories
 	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
 	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --rm --entrypoint /bin/sh $(IMAGE):build -c 'cat Dockerfile qemu.sh /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@
+	docker run --rm --entrypoint /bin/sh $(IMAGE):build -c 'cat Dockerfile /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@
 
 push: hash
 	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \

tools/qemu/qemu.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd /tmp
-
-# extract. BSD tar auto recognises compression, unlike GNU tar
-# only if stdin is a tty, if so need files volume mounted...
-[ -t 0 ] || bsdtar xzf -
-
-TGZ="$(find . -name '*.tgz' -or -name '*.tar.gz')"
-[ -n "$TGZ" ] && bsdtar xzf "$TGZ"
-
-EFI_ISO="$(find . -name '*efi.iso')"
-ISO="$(find . -name '*.iso')"
-RAW="$(find . -name '*.raw')"
-INITRD="$(find . -name '*.img')"
-KERNEL="$(find . -name vmlinuz64 -or -name '*bzImage')"
-CMDLINE="$(find . -name '*-cmdline')"
-
-if [ -n "$EFI_ISO" ]
-then
-	ARGS="-pflash /usr/share/ovmf/bios.bin -usbdevice tablet -cdrom $EFI_ISO -boot d -drive file=systemdisk.img,format=raw"
-elif [ -n "$ISO" ]
-then
-	ARGS="-cdrom $ISO -drive file=systemdisk.img,format=raw"
-elif [ -n "$RAW" ]
-then
-	# should test with more drives
-	ARGS="-drive file=$RAW,format=raw"
-elif [ -n "$KERNEL" ]
-then
-	ARGS="-kernel $KERNEL"
-	if [ -n "$INITRD" ]
-	then
-		ARGS="$ARGS -initrd $INITRD"
-	fi
-	ARGS="$ARGS -drive file=systemdisk.img,format=raw"
-else
-	echo "no recognised boot media" >2
-	exit 1
-fi
-
-echo "$ARGS" | grep -q systemdisk && qemu-img create -f raw systemdisk.img 256M
-
-if [ -n "${CMDLINE}" ]
-then
-	APPEND="$(cat $CMDLINE)"
-else
-	APPEND="$*"
-fi
-if [ -z "${APPEND}" ]
-then
-	APPEND="console=ttyS0"
-fi
-
-if [ -z "$EFI_ISO" ] && [ -z "$ISO" ]
-then
-	ARGS="-append \"${APPEND}\" ${ARGS}"
-fi
-
-eval qemu-system-x86_64 -machine q35,accel=kvm:tcg -device virtio-rng-pci -nographic -vnc none -m 1024 $ARGS