Merge pull request #3225 from rn/vmlinux

Add experimental support for building uncompressed kernels

contrib/crosvm/Dockerfile

@@ -1,7 +1,7 @@
 FROM rust:1.30.0-stretch
 
 ENV CROSVM_REPO=https://chromium.googlesource.com/chromiumos/platform/crosvm
-ENV CROSVM_COMMIT=510c783c847b6d0c18516f31fbe3dbdc782f1252
+ENV CROSVM_COMMIT=c527c1a7e8136dae1e8ae728dfd9932bf3967e7e
 ENV MINIJAIL_REPO=https://android.googlesource.com/platform/external/minijail
 ENV MINIJAIL_COMMIT=d45fc420bb8fd9d1fc9297174f3c344db8c20bbd
 

contrib/crosvm/README.md

@@ -26,35 +26,29 @@ You may also have to create an empty directory `/var/empty`.
 
 You can build a LinuxKit image suitable for `crosvm` with the
 `kernel+squashfs` build format. For example, using `minimal.yml` from
-the `./examples` directory, run:
+the `./examples` directory, run (but also see the known issues):
 
 ```sh
-linuxkit build -format kernel+squashfs minimal.yml
+linuxkit build -format kernel+squashfs -decompress-kernel minimal.yml
 ```
 
-The generated kernel file (`minimal-kernel`) needs to be converted as
+The `-vmlinux` switch is needed since `crosvm` does not grok
-`crosvm` does not grok `bzImage`s. You can convert the LinuxKit kernel
+compressed linux kernel images.
-image with
-[extract-vmlinux](https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux):
-
-```sh
-extract-vmlinux minimal-kernel > minimal-vmlinux
-```
 
 Then you can run `crosvm`:
 ```sh
-./crosvm run --seccomp-policy-dir=./seccomp/x86_64 \
+crosvm run --disable-sandbox \
     --root ./minimal-squashfs.img \
     --mem 2048 \
-    --multiprocess \
     --socket ./linuxkit-socket \
-    minimal-vmlinux
+    minimal-kernel
 ```
 
 ## Known issues
 
 - With 4.14.x, a `BUG_ON()` is hit in `drivers/base/driver.c`. 4.9.x
   kernels seem to work.
+- With the latest version, I don't seem to get a interactive console.
 - Networking does not yet work, so don't include a `onboot` `dhcpd` service.
 - `poweroff` from the command line does not work (crosvm does not seem
   to support ACPI). So to stop a VM you can use the control socket

src/cmd/linuxkit/build.go

@@ -49,6 +49,7 @@ func build(args []string) {
 	buildSize := buildCmd.String("size", "1024M", "Size for output image, if supported and fixed size")
 	buildPull := buildCmd.Bool("pull", false, "Always pull images")
 	buildDisableTrust := buildCmd.Bool("disable-content-trust", false, "Skip image trust verification specified in trust section of config (default false)")
+	buildDecompressKernel := buildCmd.Bool("decompress-kernel", false, "Decompress the Linux kernel (default false)")
 	buildCmd.Var(&buildFormats, "format", "Formats to create [ "+strings.Join(outputTypes, " ")+" ]")
 
 	if err := buildCmd.Parse(args); err != nil {
@@ -203,7 +204,7 @@ func build(args []string) {
 	if moby.Streamable(buildFormats[0]) {
 		tp = buildFormats[0]
 	}
-	err = moby.Build(m, w, *buildPull, tp)
+	err = moby.Build(m, w, *buildPull, tp, *buildDecompressKernel)
 	if err != nil {
 		log.Fatalf("%v", err)
 	}

src/cmd/linuxkit/moby/build.go

@@ -3,6 +3,8 @@ package moby
 import (
 	"archive/tar"
 	"bytes"
+	"compress/gzip"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -142,7 +144,7 @@ func outputImage(image *Image, section string, prefix string, m Moby, idMap map[
 }
 
 // Build performs the actual build process
-func Build(m Moby, w io.Writer, pull bool, tp string) error {
+func Build(m Moby, w io.Writer, pull bool, tp string, decompressKernel bool) error {
 	if MobyDir == "" {
 		MobyDir = defaultMobyConfigDir()
 	}
@@ -179,7 +181,7 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error {
 	if m.Kernel.ref != nil {
 		// get kernel and initrd tarball and ucode cpio archive from container
 		log.Infof("Extract kernel image: %s", m.Kernel.ref)
-		kf := newKernelFilter(iw, m.Kernel.Cmdline, m.Kernel.Binary, m.Kernel.Tar, m.Kernel.UCode)
+		kf := newKernelFilter(iw, m.Kernel.Cmdline, m.Kernel.Binary, m.Kernel.Tar, m.Kernel.UCode, decompressKernel)
 		err := ImageTar(m.Kernel.ref, "", kf, enforceContentTrust(m.Kernel.ref.String(), &m.Trust), pull, "")
 		if err != nil {
 			return fmt.Errorf("Failed to extract kernel image and tarball: %v", err)
@@ -255,19 +257,21 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error {
 
 // kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer
 type kernelFilter struct {
-	tw          *tar.Writer
+	tw               *tar.Writer
-	buffer      *bytes.Buffer
+	buffer           *bytes.Buffer
-	cmdline     string
+	hdr              *tar.Header
-	kernel      string
+	cmdline          string
-	tar         string
+	kernel           string
-	ucode       string
+	tar              string
-	discard     bool
+	ucode            string
-	foundKernel bool
+	decompressKernel bool
-	foundKTar   bool
+	discard          bool
-	foundUCode  bool
+	foundKernel      bool
+	foundKTar        bool
+	foundUCode       bool
 }
 
-func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode *string) *kernelFilter {
+func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode *string, decompressKernel bool) *kernelFilter {
 	tarName, kernelName, ucodeName := "kernel.tar", "kernel", ""
 	if tar != nil {
 		tarName = *tar
@@ -281,13 +285,36 @@ func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode *
 	if ucode != nil {
 		ucodeName = *ucode
 	}
-	return &kernelFilter{tw: tw, cmdline: cmdline, kernel: kernelName, tar: tarName, ucode: ucodeName}
+	return &kernelFilter{tw: tw, cmdline: cmdline, kernel: kernelName, tar: tarName, ucode: ucodeName, decompressKernel: decompressKernel}
 }
 
 func (k *kernelFilter) finishTar() error {
 	if k.buffer == nil {
 		return nil
 	}
+
+	if k.hdr != nil {
+		if k.decompressKernel {
+			log.Debugf("Decompressing kernel")
+			b, err := decompressKernel(k.buffer)
+			if err != nil {
+				return err
+			}
+			k.buffer = b
+			k.hdr.Size = int64(k.buffer.Len())
+		}
+
+		if err := k.tw.WriteHeader(k.hdr); err != nil {
+			return err
+		}
+		if _, err := k.tw.Write(k.buffer.Bytes()); err != nil {
+			return err
+		}
+		k.hdr = nil
+		k.buffer = nil
+		return nil
+	}
+
 	tr := tar.NewReader(k.buffer)
 	err := tarAppend(k.tw, tr)
 	k.buffer = nil
@@ -362,15 +389,14 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error {
 		if err != nil {
 			return err
 		}
-		whdr = &tar.Header{
+		// Stash the kernel header and prime the buffer for the kernel
+		k.hdr = &tar.Header{
 			Name:   "boot/kernel",
 			Mode:   hdr.Mode,
 			Size:   hdr.Size,
 			Format: tar.FormatPAX,
 		}
-		if err := tw.WriteHeader(whdr); err != nil {
+		k.buffer = new(bytes.Buffer)
-			return err
-		}
 	case k.tar:
 		k.foundKTar = true
 		k.discard = false
@@ -427,6 +453,85 @@ func tarAppend(iw *tar.Writer, tr *tar.Reader) error {
 	return nil
 }
 
+// Attempt to decompress a Linux kernel image
+// The kernel image can be a plain gzip'ed image (e.g., the LinuxKit arm64 kernel) or a bzImage (x86)
+// or not compressed at all (e.g., s390x). This function tries to detect the image type and decompress
+// the kernel. If no supported compressed kernel is found it returns an error.
+// For bzImages it performs some sanity checks on the header and currently only supports gzip'ed bzImages.
+func decompressKernel(src *bytes.Buffer) (*bytes.Buffer, error) {
+	const gzipMagic = "\037\213"
+
+	s := src.Bytes()
+
+	if bytes.HasPrefix(s, []byte(gzipMagic)) {
+		log.Debugf("Found gzip signature at offset: 0")
+		return gunzip(src)
+	}
+
+	// Check if it is a bzImage
+	// See: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/x86/boot.txt
+	const bzMagicIdx = 0x1fe
+	const bzMagic = uint16(0xaa55)
+	const bzHeaderIdx = 0x202
+	const bzHeader = "HdrS"
+	const bzMinLen = 0x250 // Minimum length for the required 2.0.8+ header
+	if len(s) > bzMinLen &&
+		binary.LittleEndian.Uint16(s[bzMagicIdx:bzMagicIdx+2]) == bzMagic &&
+		bytes.HasPrefix(s[bzHeaderIdx:], []byte(bzHeader)) {
+
+		log.Debugf("Found bzImage Magic and Header")
+
+		const versionIdx = 0x206
+		const setupSectorsIdx = 0x1f1
+		const sectorSize = 512
+		const payloadIdx = 0x248
+		const payloadLengthIdx = 0x24c
+
+		// Check that the version is 2.08+
+		versionMajor := int(s[versionIdx])
+		versionMinor := int(s[versionIdx+1])
+		if versionMajor < 2 && versionMinor < 8 {
+			return nil, fmt.Errorf("Unsupported bzImage version: %d.%d", versionMajor, versionMinor)
+		}
+
+		setupSectors := uint32(s[setupSectorsIdx])
+		payloadOff := binary.LittleEndian.Uint32(s[payloadIdx : payloadIdx+4])
+		payloadLen := binary.LittleEndian.Uint32(s[payloadLengthIdx : payloadLengthIdx+4])
+		payloadOff += (setupSectors + 1) * sectorSize
+		log.Debugf("bzImage: Payload at Offset: %d Length: %d", payloadOff, payloadLen)
+
+		if len(s) < int(payloadOff+payloadLen) {
+			return nil, fmt.Errorf("Compressed bzImage payload exceeds size of image")
+		}
+
+		if bytes.HasPrefix(s[payloadOff:], []byte(gzipMagic)) {
+			log.Debugf("bzImage: gzip signature at offset: %d", payloadOff)
+			return gunzip(bytes.NewBuffer(s[payloadOff : payloadOff+payloadLen]))
+		}
+		// TODO(rn): Add more supported formats
+		return nil, fmt.Errorf("Unsupported bzImage payload format at offset %d", payloadOff)
+	}
+
+	return nil, fmt.Errorf("No compressed kernel or no supported format found")
+}
+
+func gunzip(src *bytes.Buffer) (*bytes.Buffer, error) {
+	dst := new(bytes.Buffer)
+
+	zr, err := gzip.NewReader(src)
+	if err != nil {
+		return nil, err
+	}
+
+	n, err := io.Copy(dst, zr)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf("gunzip'ed %d bytes", n)
+	return dst, nil
+}
+
 // this allows inserting metadata into a file in the image
 func metadata(m Moby, md string) ([]byte, error) {
 	// Make sure the Image strings are update to date with the refs

src/cmd/linuxkit/moby/linuxkit.go

@@ -62,7 +62,7 @@ func ensureLinuxkitImage(name string) error {
 		return err
 	}
 	defer os.Remove(tf.Name())
-	Build(m, tf, false, "")
+	Build(m, tf, false, "", false)
 	if err := tf.Close(); err != nil {
 		return err
 	}