From 0ba381c77a740cd34a7de545b4b1a4dfd6947201 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 20 Apr 2017 13:31:48 +0100 Subject: [PATCH 1/4] tools: Change qemu image for use with moby run - Remove the qemu.sh entrypoint - This means that any qemu command can be used (e.g qemu.img) Signed-off-by: Dave Tucker --- tools/qemu/Dockerfile | 1 - tools/qemu/Makefile | 4 +-- tools/qemu/qemu.sh | 62 ------------------------------------------- 3 files changed, 2 insertions(+), 65 deletions(-) delete mode 100755 tools/qemu/qemu.sh diff --git a/tools/qemu/Dockerfile b/tools/qemu/Dockerfile index 8bd33fd64..fdf734836 100644 --- a/tools/qemu/Dockerfile +++ b/tools/qemu/Dockerfile @@ -13,4 +13,3 @@ RUN \ && true COPY . . -ENTRYPOINT ["/qemu.sh"] diff --git a/tools/qemu/Makefile b/tools/qemu/Makefile index d28f5c113..081ab654f 100644 --- a/tools/qemu/Makefile +++ b/tools/qemu/Makefile @@ -5,10 +5,10 @@ IMAGE=qemu default: push -hash: Dockerfile qemu.sh repositories +hash: Dockerfile repositories DOCKER_CONTENT_TRUST=1 docker pull $(BASE) tar cf - $^ | docker build --no-cache -t $(IMAGE):build - - docker run --rm --entrypoint /bin/sh $(IMAGE):build -c 'cat Dockerfile qemu.sh /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@ + docker run --rm --entrypoint /bin/sh $(IMAGE):build -c 'cat Dockerfile /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@ push: hash docker pull linuxkit/$(IMAGE):$(shell cat hash) || \ diff --git a/tools/qemu/qemu.sh b/tools/qemu/qemu.sh deleted file mode 100755 index a301e38de..000000000 --- a/tools/qemu/qemu.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh - -set -e - -cd /tmp - -# extract. BSD tar auto recognises compression, unlike GNU tar -# only if stdin is a tty, if so need files volume mounted... -[ -t 0 ] || bsdtar xzf - - -TGZ="$(find . -name '*.tgz' -or -name '*.tar.gz')" -[ -n "$TGZ" ] && bsdtar xzf "$TGZ" - -EFI_ISO="$(find . -name '*efi.iso')" -ISO="$(find . -name '*.iso')" -RAW="$(find . -name '*.raw')" -INITRD="$(find . -name '*.img')" -KERNEL="$(find . -name vmlinuz64 -or -name '*bzImage')" -CMDLINE="$(find . -name '*-cmdline')" - -if [ -n "$EFI_ISO" ] -then - ARGS="-pflash /usr/share/ovmf/bios.bin -usbdevice tablet -cdrom $EFI_ISO -boot d -drive file=systemdisk.img,format=raw" -elif [ -n "$ISO" ] -then - ARGS="-cdrom $ISO -drive file=systemdisk.img,format=raw" -elif [ -n "$RAW" ] -then - # should test with more drives - ARGS="-drive file=$RAW,format=raw" -elif [ -n "$KERNEL" ] -then - ARGS="-kernel $KERNEL" - if [ -n "$INITRD" ] - then - ARGS="$ARGS -initrd $INITRD" - fi - ARGS="$ARGS -drive file=systemdisk.img,format=raw" -else - echo "no recognised boot media" >2 - exit 1 -fi - -echo "$ARGS" | grep -q systemdisk && qemu-img create -f raw systemdisk.img 256M - -if [ -n "${CMDLINE}" ] -then - APPEND="$(cat $CMDLINE)" -else - APPEND="$*" -fi -if [ -z "${APPEND}" ] -then - APPEND="console=ttyS0" -fi - -if [ -z "$EFI_ISO" ] && [ -z "$ISO" ] -then - ARGS="-append \"${APPEND}\" ${ARGS}" -fi - -eval qemu-system-x86_64 -machine q35,accel=kvm:tcg -device virtio-rng-pci -nographic -vnc none -m 1024 $ARGS From 48daeda07e3b22521783c6ac5e5cfe78c757424c Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 20 Apr 2017 13:39:54 +0100 Subject: [PATCH 2/4] moby: Add container fallback for moby run qemu This commit allows moby run qemu to fallback to using a container if qemu isn't installed on the host OS Signed-off-by: Dave Tucker --- src/cmd/moby/run_qemu.go | 242 +++++++++++++++++++++++++++++---------- 1 file changed, 182 insertions(+), 60 deletions(-) diff --git a/src/cmd/moby/run_qemu.go b/src/cmd/moby/run_qemu.go index bdc1543ce..c0d515890 100644 --- a/src/cmd/moby/run_qemu.go +++ b/src/cmd/moby/run_qemu.go @@ -10,6 +10,28 @@ import ( log "github.com/Sirupsen/logrus" ) +// QemuImg is the version of qemu container +const QemuImg = "linuxkit/qemu:17f052263d63c8a2b641ad91c589edcbb8a18c82" + +// QemuConfig contains the config for Qemu +type QemuConfig struct { + Prefix string + ISO bool + UEFI bool + Kernel bool + GUI bool + DiskPath string + DiskSize string + FWPath string + Arch string + CPUs string + Memory string + KVM bool + Containerized bool + QemuBinPath string + QemuImgPath string +} + func runQemu(args []string) { qemuFlags := flag.NewFlagSet("qemu", flag.ExitOnError) qemuFlags.Usage = func() { @@ -53,78 +75,192 @@ func runQemu(args []string) { log.Warnf("Both -iso and -uefi have been used") } - // Before building qemu arguments, check qemu is in the $PATH - qemuBinPath := "qemu-system-" + *qemuArch - qemuImgPath := "qemu-img" - fullQemuPath, err := exec.LookPath(qemuBinPath) + config := QemuConfig{ + Prefix: prefix, + ISO: *qemuIso, + UEFI: *qemuUEFI, + Kernel: *qemuKernel, + GUI: *qemuGUI, + DiskPath: *qemuDiskPath, + DiskSize: *qemuDiskSize, + FWPath: *qemuFWPath, + Arch: *qemuArch, + CPUs: *qemuCPUs, + Memory: *qemuMem, + } + + config, qemuArgs := buildQemuCmdline(config) + + var err error + if config.Containerized { + err = runQemuContainer(config, qemuArgs) + } else { + err = runQemuLocal(config, qemuArgs) + } if err != nil { - log.Fatalf("Unable to find %s within the $PATH", qemuBinPath) + log.Fatal(err.Error()) + } +} + +func runQemuLocal(config QemuConfig, args []string) error { + if config.DiskPath != "" { + // If disk doesn't exist then create one + if _, err := os.Stat(config.DiskPath); err != nil { + if os.IsNotExist(err) { + log.Infof("Creating new qemu disk [%s]", config.DiskPath) + qemuImgCmd := exec.Command(config.QemuImgPath, "create", "-f", "qcow2", config.DiskPath, config.DiskSize) + log.Debugf("%v\n", qemuImgCmd.Args) + if err := qemuImgCmd.Run(); err != nil { + return fmt.Errorf("Error creating disk [%s]: %s", config.DiskPath, err.Error()) + } + } else { + return err + } + } else { + log.Infof("Using existing disk [%s]", config.DiskPath) + } + } + + // Check for OVMF firmware before running + if config.UEFI { + if _, err := os.Stat(config.FWPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("File [%s] does not exist, please ensure OVMF is installed", config.FWPath) + } + return err + } + } + + qemuCmd := exec.Command(config.QemuBinPath, args...) + // If verbosity is enabled print out the full path/arguments + log.Debugf("%v\n", qemuCmd.Args) + + // If we're not using a separate window then link the execution to stdin/out + if config.GUI != true { + qemuCmd.Stdin = os.Stdin + qemuCmd.Stdout = os.Stdout + qemuCmd.Stderr = os.Stderr + } + + return qemuCmd.Run() +} + +func runQemuContainer(config QemuConfig, args []string) error { + wd, err := os.Getwd() + if err != nil { + return err + } + + dockerArgs := []string{"run", "-i", "--rm", "-v", fmt.Sprintf("%s:%s", wd, "/tmp"), "-w", "/tmp"} + + if config.KVM { + dockerArgs = append(dockerArgs, "--device", "/dev/kvm") + } + + dockerPath, err := exec.LookPath("docker") + if err != nil { + return fmt.Errorf("Unable to find docker in the $PATH") + } + + if config.DiskPath != "" { + // If disk doesn't exist then create one + if _, err = os.Stat(config.DiskPath); err != nil { + if os.IsNotExist(err) { + log.Infof("Creating new qemu disk [%s]", config.DiskPath) + imgArgs := append(dockerArgs, QemuImg, "qemu-img", "create", "-f", "qcow2", config.DiskPath, config.DiskSize) + qemuImgCmd := exec.Command(dockerPath, imgArgs...) + log.Debugf("%v\n", qemuImgCmd.Args) + if err = qemuImgCmd.Run(); err != nil { + return fmt.Errorf("Error creating disk [%s]: %s", config.DiskPath, err.Error()) + } + } else { + return err + } + } else { + log.Infof("Using existing disk [%s]", config.DiskPath) + } + } + + qemuArgs := append(dockerArgs, QemuImg, "qemu-system-"+config.Arch) + qemuArgs = append(qemuArgs, args...) + qemuCmd := exec.Command(dockerPath, qemuArgs...) + // If verbosity is enabled print out the full path/arguments + log.Debugf("%v\n", qemuCmd.Args) + + // GUI mode not currently supported in a container. Although it could be in future. + if config.GUI == true { + return fmt.Errorf("GUI mode is only supported when running locally, not in a container") + } + + qemuCmd.Stdin = os.Stdin + qemuCmd.Stdout = os.Stdout + qemuCmd.Stderr = os.Stderr + + return qemuCmd.Run() +} + +func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { + // Before building qemu arguments, check if qemu is in the PATH or fallback to containerized + qemuBinPath := "qemu-system-" + config.Arch + qemuImgPath := "qemu-img" + + var err error + config.QemuBinPath, err = exec.LookPath(qemuBinPath) + if err != nil { + log.Infof("Unable to find %s within the $PATH. Using a container", qemuBinPath) + config.Containerized = true + } + + config.QemuImgPath, err = exec.LookPath(qemuImgPath) + if err != nil { + // No need to show the error message twice + if !config.Containerized { + log.Infof("Unable to find %s within the $PATH. Using a container", qemuImgPath) + config.Containerized = true + } } // Iterate through the flags and build arguments var qemuArgs []string qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci") - qemuArgs = append(qemuArgs, "-smp", *qemuCPUs) - qemuArgs = append(qemuArgs, "-m", *qemuMem) + qemuArgs = append(qemuArgs, "-smp", config.CPUs) + qemuArgs = append(qemuArgs, "-m", config.Memory) // Look for kvm device and enable for qemu if it exists if _, err = os.Stat("/dev/kvm"); os.IsNotExist(err) { qemuArgs = append(qemuArgs, "-machine", "q35") } else { + config.KVM = true qemuArgs = append(qemuArgs, "-enable-kvm") qemuArgs = append(qemuArgs, "-machine", "q35,accel=kvm:tcg") } - if *qemuDiskPath != "" { - // If disk doesn't exist then create one - if _, err = os.Stat(*qemuDiskPath); os.IsNotExist(err) { - log.Infof("Creating new qemu disk [%s]", *qemuDiskPath) - fullQemuImgPath, err := exec.LookPath(qemuImgPath) - if err != nil { - log.Fatalf("Unable to find %s within the $PATH", qemuImgPath) - } - cmd := exec.Command(fullQemuImgPath, "create", "-f", "qcow2", *qemuDiskPath, *qemuDiskSize) - if err = cmd.Run(); err != nil { - log.Fatalf("Error creating disk [%s]: %s", *qemuDiskPath, err.Error()) - } - } else { - log.Infof("Using existing disk [%s]", *qemuDiskPath) - } - qemuArgs = append(qemuArgs, "-drive", "file="+*qemuDiskPath+",format=qcow2") + if config.DiskPath != "" { + qemuArgs = append(qemuArgs, "-drive", "file="+config.DiskPath+",format=qcow2") } // Check flags for iso/uefi boot and if so disable kernel boot - if *qemuIso { - *qemuKernel = false - qemuIsoPath := buildPath(prefix, ".iso") + if config.ISO { + config.Kernel = false + qemuIsoPath := buildPath(config.Prefix, ".iso") qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath) } - if *qemuUEFI { - // Check for OVMF firmware before building paths - _, err = os.Stat(*qemuFWPath) - if err != nil { - if os.IsNotExist(err) { - log.Fatalf("File [%s] does not exist, please ensure OVMF is installed", *qemuFWPath) - } else { - log.Fatalf("%s", err.Error()) - } - } - - *qemuKernel = false - qemuIsoPath := buildPath(prefix, "-efi.iso") - qemuArgs = append(qemuArgs, "-pflash", *qemuFWPath) + if config.UEFI { + config.Kernel = false + qemuIsoPath := buildPath(config.Prefix, "-efi.iso") + qemuArgs = append(qemuArgs, "-pflash", config.FWPath) qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath) qemuArgs = append(qemuArgs, "-boot", "d") } // build kernel boot config from bzImage/initrd/cmdline - if *qemuKernel { - qemuKernelPath := buildPath(prefix, "-bzImage") - qemuInitrdPath := buildPath(prefix, "-initrd.img") + if config.Kernel { + qemuKernelPath := buildPath(config.Prefix, "-bzImage") + qemuInitrdPath := buildPath(config.Prefix, "-initrd.img") qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath) qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath) - consoleString, err := ioutil.ReadFile(prefix + "-cmdline") + consoleString, err := ioutil.ReadFile(config.Prefix + "-cmdline") if err != nil { log.Infof(" %s\n defaulting to console output", err.Error()) qemuArgs = append(qemuArgs, "-append", "console=ttyS0 console=tty0 page_poison=1") @@ -133,25 +269,11 @@ func runQemu(args []string) { } } - if *qemuGUI != true { + if config.GUI != true { qemuArgs = append(qemuArgs, "-nographic") } - // If verbosity is enabled print out the full path/arguments - log.Debugf("%s %v\n", fullQemuPath, qemuArgs) - - cmd := exec.Command(fullQemuPath, qemuArgs...) - - // If we're not using a seperate window then link the execution to stdin/out - if *qemuGUI != true { - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - if err = cmd.Run(); err != nil { - log.Fatalf("Error starting %s: %s", fullQemuPath, err.Error()) - } + return config, qemuArgs } func buildPath(prefix string, postfix string) string { From 11fcf087ca8aa112c6d5abcde4959286023a028b Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 20 Apr 2017 13:40:28 +0100 Subject: [PATCH 3/4] makefile: Use moby run qemu This deprectes scripts/qemu.sh as moby run qemu can now safely be run in CI, where we currently used the qemu container Signed-off-by: Dave Tucker --- Makefile | 4 ++-- scripts/qemu.sh | 47 ----------------------------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) delete mode 100755 scripts/qemu.sh diff --git a/Makefile b/Makefile index 6552ccd94..56a7e43c7 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test-bzImage: test-initrd.img # interactive versions need to use volume mounts .PHONY: test-qemu-efi test-qemu-efi: test-efi.iso - ./scripts/qemu.sh $^ 2>&1 | tee test-efi.log + script -q /dev/null $(MOBY) run $^ | tee test-efi.log $(call check_test_log, test-efi.log) bin: @@ -68,7 +68,7 @@ test-gcp: $(MOBY) test.img.tar.gz .PHONY: test test: test-initrd.img test-bzImage test-cmdline - tar cf - $^ | ./scripts/qemu.sh 2>&1 | tee test.log + script -q /dev/null $(MOBY) run test | tee test.log $(call check_test_log, test.log) test-ltp.img.tar.gz: $(MOBY) test/ltp/test-ltp.yml diff --git a/scripts/qemu.sh b/scripts/qemu.sh deleted file mode 100755 index f7b59bf8f..000000000 --- a/scripts/qemu.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -QEMU_IMAGE=linuxkit/qemu:4563d58e97958f4941fbef9e74cabc08bd402144@sha256:b2db0b13ba1cbb6b48218f088fe0a4d860e1db2c4c6381b5416536f48a612230 - -# if not interactive -if [ ! -t 0 -a -z "$1" ] -then - # non interactive, tarball input - docker run -i --rm "$QEMU_IMAGE" - exit $? -fi - -FILE=$1 -FILE2=$2 -CMDLINE=$3 -[ -z "$FILE" ] && FILE="$PWD/moby" - -BASE=$(basename "$FILE") -DIR=$(dirname "$FILE") -if [ ! -f "$FILE" -a -f $DIR/$BASE-initrd.img -a -f $DIR/$BASE-bzImage ] -then - FILE=$DIR/$BASE-initrd.img - FILE2=$DIR/$BASE-bzImage -fi - -echo "$FILE" | grep -q '^/' || FILE="$PWD/$FILE" -if [ ! -z "$FILE2" ] -then - echo "$FILE2" | grep -q '^/' || FILE2="$PWD/$FILE2" -fi -if [ ! -z "$CMDLINE" ] -then - echo "$CMDLINE" | grep -q '^/' || CMDLINE="$PWD/$CMDLINE" -fi - -if [ -c "/dev/kvm" ] ; then - DEVKVM="--device=/dev/kvm" -fi -BASE=$(basename "$FILE") -MOUNTS="-v $FILE:/tmp/$BASE" -BASE2=$(basename "$FILE2") -BASE3=$(basename "$CMDLINE") - -[ ! -z "$FILE2" ] && MOUNTS="$MOUNTS -v $FILE2:/tmp/$BASE2" -[ ! -z "$CMDLINE" ] && MOUNTS="$MOUNTS -v $CMDLINE:/tmp/$BASE3" - -docker run -it --rm $MOUNTS $DEVKVM "$QEMU_IMAGE" From 745dcb66b90f2ffc0807d40fead0f76f50ff0c20 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 20 Apr 2017 14:48:12 +0100 Subject: [PATCH 4/4] makefile: Remove script -q so errors appear in CI Signed-off-by: Dave Tucker --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 56a7e43c7..9119ac49e 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test-bzImage: test-initrd.img # interactive versions need to use volume mounts .PHONY: test-qemu-efi test-qemu-efi: test-efi.iso - script -q /dev/null $(MOBY) run $^ | tee test-efi.log + $(MOBY) run $^ | tee test-efi.log $(call check_test_log, test-efi.log) bin: @@ -58,17 +58,17 @@ endef .PHONY: test-hyperkit test-hyperkit: $(MOBY) test-initrd.img test-bzImage test-cmdline rm -f disk.img - script -q /dev/null $(MOBY) run test | tee test.log + $(MOBY) run test | tee test.log $(call check_test_log, test.log) .PHONY: test-gcp test-gcp: $(MOBY) test.img.tar.gz - script -q /dev/null $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log + $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log $(call check_test_log, test-gcp.log) .PHONY: test test: test-initrd.img test-bzImage test-cmdline - script -q /dev/null $(MOBY) run test | tee test.log + $(MOBY) run test | tee test.log $(call check_test_log, test.log) test-ltp.img.tar.gz: $(MOBY) test/ltp/test-ltp.yml