From 4f0cec5c140a0eba9d2078c51873dc0558bfb934 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Sat, 3 Nov 2018 21:40:29 +0000 Subject: [PATCH 1/5] build: Restructure the kernel filter Stash the kernel image in a local buffer and flush it out once done. This is preparation work for supporting uncompressed kernels in the next commit. Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/moby/build.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/cmd/linuxkit/moby/build.go b/src/cmd/linuxkit/moby/build.go index 284a36db3..932e6d8d4 100644 --- a/src/cmd/linuxkit/moby/build.go +++ b/src/cmd/linuxkit/moby/build.go @@ -257,6 +257,7 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error { type kernelFilter struct { tw *tar.Writer buffer *bytes.Buffer + hdr *tar.Header cmdline string kernel string tar string @@ -288,6 +289,19 @@ func (k *kernelFilter) finishTar() error { if k.buffer == nil { return nil } + + if k.hdr != nil { + if err := k.tw.WriteHeader(k.hdr); err != nil { + return err + } + if _, err := k.tw.Write(k.buffer.Bytes()); err != nil { + return err + } + k.hdr = nil + k.buffer = nil + return nil + } + tr := tar.NewReader(k.buffer) err := tarAppend(k.tw, tr) k.buffer = nil @@ -362,15 +376,14 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { if err != nil { return err } - whdr = &tar.Header{ + // Stash the kernel header and prime the buffer for the kernel + k.hdr = &tar.Header{ Name: "boot/kernel", Mode: hdr.Mode, Size: hdr.Size, Format: tar.FormatPAX, } - if err := tw.WriteHeader(whdr); err != nil { - return err - } + k.buffer = new(bytes.Buffer) case k.tar: k.foundKTar = true k.discard = false From 09fbcb59d7cacef12c79080f7454fc7c3cf1f460 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Sat, 10 Nov 2018 14:43:50 +0000 Subject: [PATCH 2/5] cmd: Add scaffolding to decompress the kernel Add the '-vmlinux' flag to build and pass it all the way to the kernel filter. Note, this commit only adds the flag but does not yet perform the decompression. This will be added with the next commit. Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/build.go | 3 ++- src/cmd/linuxkit/moby/build.go | 31 ++++++++++++++++--------------- src/cmd/linuxkit/moby/linuxkit.go | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/cmd/linuxkit/build.go b/src/cmd/linuxkit/build.go index 6c991a15e..37a00bbac 100644 --- a/src/cmd/linuxkit/build.go +++ b/src/cmd/linuxkit/build.go @@ -49,6 +49,7 @@ func build(args []string) { buildSize := buildCmd.String("size", "1024M", "Size for output image, if supported and fixed size") buildPull := buildCmd.Bool("pull", false, "Always pull images") buildDisableTrust := buildCmd.Bool("disable-content-trust", false, "Skip image trust verification specified in trust section of config (default false)") + buildDecompressKernel := buildCmd.Bool("decompress-kernel", false, "Decompress the Linux kernel (default false)") buildCmd.Var(&buildFormats, "format", "Formats to create [ "+strings.Join(outputTypes, " ")+" ]") if err := buildCmd.Parse(args); err != nil { @@ -203,7 +204,7 @@ func build(args []string) { if moby.Streamable(buildFormats[0]) { tp = buildFormats[0] } - err = moby.Build(m, w, *buildPull, tp) + err = moby.Build(m, w, *buildPull, tp, *buildDecompressKernel) if err != nil { log.Fatalf("%v", err) } diff --git a/src/cmd/linuxkit/moby/build.go b/src/cmd/linuxkit/moby/build.go index 932e6d8d4..8f307557a 100644 --- a/src/cmd/linuxkit/moby/build.go +++ b/src/cmd/linuxkit/moby/build.go @@ -142,7 +142,7 @@ func outputImage(image *Image, section string, prefix string, m Moby, idMap map[ } // Build performs the actual build process -func Build(m Moby, w io.Writer, pull bool, tp string) error { +func Build(m Moby, w io.Writer, pull bool, tp string, decompressKernel bool) error { if MobyDir == "" { MobyDir = defaultMobyConfigDir() } @@ -179,7 +179,7 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error { if m.Kernel.ref != nil { // get kernel and initrd tarball and ucode cpio archive from container log.Infof("Extract kernel image: %s", m.Kernel.ref) - kf := newKernelFilter(iw, m.Kernel.Cmdline, m.Kernel.Binary, m.Kernel.Tar, m.Kernel.UCode) + kf := newKernelFilter(iw, m.Kernel.Cmdline, m.Kernel.Binary, m.Kernel.Tar, m.Kernel.UCode, decompressKernel) err := ImageTar(m.Kernel.ref, "", kf, enforceContentTrust(m.Kernel.ref.String(), &m.Trust), pull, "") if err != nil { return fmt.Errorf("Failed to extract kernel image and tarball: %v", err) @@ -255,20 +255,21 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error { // kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer type kernelFilter struct { - tw *tar.Writer - buffer *bytes.Buffer - hdr *tar.Header - cmdline string - kernel string - tar string - ucode string - discard bool - foundKernel bool - foundKTar bool - foundUCode bool + tw *tar.Writer + buffer *bytes.Buffer + hdr *tar.Header + cmdline string + kernel string + tar string + ucode string + decompressKernel bool + discard bool + foundKernel bool + foundKTar bool + foundUCode bool } -func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode *string) *kernelFilter { +func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode *string, decompressKernel bool) *kernelFilter { tarName, kernelName, ucodeName := "kernel.tar", "kernel", "" if tar != nil { tarName = *tar @@ -282,7 +283,7 @@ func newKernelFilter(tw *tar.Writer, cmdline string, kernel string, tar, ucode * if ucode != nil { ucodeName = *ucode } - return &kernelFilter{tw: tw, cmdline: cmdline, kernel: kernelName, tar: tarName, ucode: ucodeName} + return &kernelFilter{tw: tw, cmdline: cmdline, kernel: kernelName, tar: tarName, ucode: ucodeName, decompressKernel: decompressKernel} } func (k *kernelFilter) finishTar() error { diff --git a/src/cmd/linuxkit/moby/linuxkit.go b/src/cmd/linuxkit/moby/linuxkit.go index c77764ab9..56cde4619 100644 --- a/src/cmd/linuxkit/moby/linuxkit.go +++ b/src/cmd/linuxkit/moby/linuxkit.go @@ -62,7 +62,7 @@ func ensureLinuxkitImage(name string) error { return err } defer os.Remove(tf.Name()) - Build(m, tf, false, "") + Build(m, tf, false, "", false) if err := tf.Close(); err != nil { return err } From f635cad7a61998c0e359898ab0f40b8f1aee0f42 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Mon, 12 Nov 2018 21:28:36 +0000 Subject: [PATCH 3/5] build: Add support for decompressing bzLinux kernels Support plain gzip'ed files, as used on arm64, and bzImage with embedded gzip'ed kernel, as used on x86. Signed-off-by: Rolf Neugebauer --- src/cmd/linuxkit/moby/build.go | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/src/cmd/linuxkit/moby/build.go b/src/cmd/linuxkit/moby/build.go index 8f307557a..484aa89d3 100644 --- a/src/cmd/linuxkit/moby/build.go +++ b/src/cmd/linuxkit/moby/build.go @@ -3,6 +3,8 @@ package moby import ( "archive/tar" "bytes" + "compress/gzip" + "encoding/binary" "encoding/json" "errors" "fmt" @@ -292,6 +294,16 @@ func (k *kernelFilter) finishTar() error { } if k.hdr != nil { + if k.decompressKernel { + log.Debugf("Decompressing kernel") + b, err := decompressKernel(k.buffer) + if err != nil { + return err + } + k.buffer = b + k.hdr.Size = int64(k.buffer.Len()) + } + if err := k.tw.WriteHeader(k.hdr); err != nil { return err } @@ -441,6 +453,85 @@ func tarAppend(iw *tar.Writer, tr *tar.Reader) error { return nil } +// Attempt to decompress a Linux kernel image +// The kernel image can be a plain gzip'ed image (e.g., the LinuxKit arm64 kernel) or a bzImage (x86) +// or not compressed at all (e.g., s390x). This function tries to detect the image type and decompress +// the kernel. If no supported compressed kernel is found it returns an error. +// For bzImages it performs some sanity checks on the header and currently only supports gzip'ed bzImages. +func decompressKernel(src *bytes.Buffer) (*bytes.Buffer, error) { + const gzipMagic = "\037\213" + + s := src.Bytes() + + if bytes.HasPrefix(s, []byte(gzipMagic)) { + log.Debugf("Found gzip signature at offset: 0") + return gunzip(src) + } + + // Check if it is a bzImage + // See: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/x86/boot.txt + const bzMagicIdx = 0x1fe + const bzMagic = uint16(0xaa55) + const bzHeaderIdx = 0x202 + const bzHeader = "HdrS" + const bzMinLen = 0x250 // Minimum length for the required 2.0.8+ header + if len(s) > bzMinLen && + binary.LittleEndian.Uint16(s[bzMagicIdx:bzMagicIdx+2]) == bzMagic && + bytes.HasPrefix(s[bzHeaderIdx:], []byte(bzHeader)) { + + log.Debugf("Found bzImage Magic and Header") + + const versionIdx = 0x206 + const setupSectorsIdx = 0x1f1 + const sectorSize = 512 + const payloadIdx = 0x248 + const payloadLengthIdx = 0x24c + + // Check that the version is 2.08+ + versionMajor := int(s[versionIdx]) + versionMinor := int(s[versionIdx+1]) + if versionMajor < 2 && versionMinor < 8 { + return nil, fmt.Errorf("Unsupported bzImage version: %d.%d", versionMajor, versionMinor) + } + + setupSectors := uint32(s[setupSectorsIdx]) + payloadOff := binary.LittleEndian.Uint32(s[payloadIdx : payloadIdx+4]) + payloadLen := binary.LittleEndian.Uint32(s[payloadLengthIdx : payloadLengthIdx+4]) + payloadOff += (setupSectors + 1) * sectorSize + log.Debugf("bzImage: Payload at Offset: %d Length: %d", payloadOff, payloadLen) + + if len(s) < int(payloadOff+payloadLen) { + return nil, fmt.Errorf("Compressed bzImage payload exceeds size of image") + } + + if bytes.HasPrefix(s[payloadOff:], []byte(gzipMagic)) { + log.Debugf("bzImage: gzip signature at offset: %d", payloadOff) + return gunzip(bytes.NewBuffer(s[payloadOff : payloadOff+payloadLen])) + } + // TODO(rn): Add more supported formats + return nil, fmt.Errorf("Unsupported bzImage payload format at offset %d", payloadOff) + } + + return nil, fmt.Errorf("No compressed kernel or no supported format found") +} + +func gunzip(src *bytes.Buffer) (*bytes.Buffer, error) { + dst := new(bytes.Buffer) + + zr, err := gzip.NewReader(src) + if err != nil { + return nil, err + } + + n, err := io.Copy(dst, zr) + if err != nil { + return nil, err + } + + log.Debugf("gunzip'ed %d bytes", n) + return dst, nil +} + // this allows inserting metadata into a file in the image func metadata(m Moby, md string) ([]byte, error) { // Make sure the Image strings are update to date with the refs From fa719063a8bd112163d71af3ecd50f70fe36e2c3 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Wed, 14 Nov 2018 23:03:52 +0000 Subject: [PATCH 4/5] contrib: Update crosvm to latest version Signed-off-by: Rolf Neugebauer --- contrib/crosvm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/crosvm/Dockerfile b/contrib/crosvm/Dockerfile index 8c8bc57aa..5da9b9039 100644 --- a/contrib/crosvm/Dockerfile +++ b/contrib/crosvm/Dockerfile @@ -1,7 +1,7 @@ FROM rust:1.30.0-stretch ENV CROSVM_REPO=https://chromium.googlesource.com/chromiumos/platform/crosvm -ENV CROSVM_COMMIT=510c783c847b6d0c18516f31fbe3dbdc782f1252 +ENV CROSVM_COMMIT=c527c1a7e8136dae1e8ae728dfd9932bf3967e7e ENV MINIJAIL_REPO=https://android.googlesource.com/platform/external/minijail ENV MINIJAIL_COMMIT=d45fc420bb8fd9d1fc9297174f3c344db8c20bbd From f1667aac67860cd8c00940175ed759042dffcd34 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Wed, 14 Nov 2018 23:15:56 +0000 Subject: [PATCH 5/5] contrib: Update crosvm README Signed-off-by: Rolf Neugebauer --- contrib/crosvm/README.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/contrib/crosvm/README.md b/contrib/crosvm/README.md index 3601e192c..dd09260cc 100644 --- a/contrib/crosvm/README.md +++ b/contrib/crosvm/README.md @@ -26,35 +26,29 @@ You may also have to create an empty directory `/var/empty`. You can build a LinuxKit image suitable for `crosvm` with the `kernel+squashfs` build format. For example, using `minimal.yml` from -the `./examples` directory, run: +the `./examples` directory, run (but also see the known issues): ```sh -linuxkit build -format kernel+squashfs minimal.yml +linuxkit build -format kernel+squashfs -decompress-kernel minimal.yml ``` -The generated kernel file (`minimal-kernel`) needs to be converted as -`crosvm` does not grok `bzImage`s. You can convert the LinuxKit kernel -image with -[extract-vmlinux](https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux): - -```sh -extract-vmlinux minimal-kernel > minimal-vmlinux -``` +The `-vmlinux` switch is needed since `crosvm` does not grok +compressed linux kernel images. Then you can run `crosvm`: ```sh -./crosvm run --seccomp-policy-dir=./seccomp/x86_64 \ +crosvm run --disable-sandbox \ --root ./minimal-squashfs.img \ --mem 2048 \ - --multiprocess \ --socket ./linuxkit-socket \ - minimal-vmlinux + minimal-kernel ``` ## Known issues - With 4.14.x, a `BUG_ON()` is hit in `drivers/base/driver.c`. 4.9.x kernels seem to work. +- With the latest version, I don't seem to get a interactive console. - Networking does not yet work, so don't include a `onboot` `dhcpd` service. - `poweroff` from the command line does not work (crosvm does not seem to support ACPI). So to stop a VM you can use the control socket