diff --git a/README.md b/README.md index 6884dfe55..4dfb023c5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ LinuxKit, a toolkit for building custom minimal, immutable Linux distributions. - Completely stateless, but persistent storage can be attached - Easy tooling, with easy iteration - Built with containers, for running containers +- Designed to create [reproducible builds](./docs/reproducible-builds.md) [WIP] - Designed for building and running clustered applications, including but not limited to container orchestration such as Docker or Kubernetes - Designed from the experience of building Docker Editions, but redesigned as a general-purpose toolkit - Designed to be managed by external tooling, such as [Infrakit](https://github.com/docker/infrakit) or similar tools @@ -155,7 +156,7 @@ This is an open project without fixed judgements, open to the community to set t ## Development reports -There are weekly [development reports](reports/) summarizing work carried out in the week. +There are monthly [development reports](reports/) summarising the work carried out each month. ## Adopters diff --git a/docs/reproducible-builds.md b/docs/reproducible-builds.md new file mode 100644 index 000000000..4824dfe4e --- /dev/null +++ b/docs/reproducible-builds.md @@ -0,0 +1,71 @@ +# Reproducible builds + +We aim to make the outputs of `linuxkit build` reproducible, i.e. the +build artefacts should be bit-by-bit identical copies if invoked with +the same inputs and run with the same version of the `linuxkit` +command. See [this +document](https://reproducible-builds.org/docs/buy-in/) on why this +matters. + +_Note, we do not (yet) aim to make `linuxkit pkg build` builds +reproducible._ + + +## Current status + +Currently, the following output formats provide reproducible builds: +- `tar` (Tested as part of the CI) +- `tar-kernel-initrd` +- `docker` +- `kernel+initrd` (Tested as part of the CI) + + +## Details + +In general, `linuxkit build` lends itself for reproducible +builds. LinuxKit packages, used during `linuxkit build`, are (signed) +docker images. Packages are tagged with the content hash of the source +code (and optionally release version) and are typically only updated +if the source of the package changed (in which case the tag +changes). For all intents and purposes, when pulled by tag, the +contents of a packages should be bit-by-bit identical. Alternatively, +the digest of the package, in which case, the pulled image will always +be the same. + +The first phase of the `linuxkit build` mostly untars and retars the +images of the packages to produce an tar file of the root filesystem. +This then serves as input for other output formats. During this first +phase, there are a number of things to watch out for to generate +reproducible builds: + +- Timestamps of generated files. The `docker export` command, as well + as `linuxkit build` itself, creates a small number of files. The + `ModTime` for these files needs to be clamped to a fixed date + (otherwise the current time is used). Use the `defaultModTime` + variable to set the `ModTime` of created files to a specific time. +- Generated JSON files. `linuxkit build` generates a number of JSON + files by marshalling Go `struct` variables. Examples are the OCI + specification `config.json` and `runtime.json` files for + containers. The default Go `json.Marshal()` function seems to do a + reasonable good job in generating reproducible output from internal + structures, including for JSON objects. However, during `linuxkit + build` some of the OCI runtime spec fields are generated/modified + and care must be taken to ensure consistent ordering. For JSON + arrays (Go slices) it is best to sort them before Marshalling them. + +Reproducible builds for the first phase of `linuxkit build` can be +tested using `-output tar` and comparing the output of subsequent +builds with tools like `diff` or the excellent +[`diffoscope`](https://diffoscope.org/). + +The second phase of `linuxkit build` converts the intermediary `tar` +format into the desired output format. Making this phase reproducible +depends on the tools used to generate the output. + +Builds, which produce ISO formats should probably be converted to use +[`go-diskfs`](https://github.com/diskfs/go-diskfs) before attempting +to make them reproducible. + +For ideas on how to make the builds for other output formats +reproducible, see [this +page](https://reproducible-builds.org/docs/system-images/). diff --git a/src/cmd/linuxkit/moby/build.go b/src/cmd/linuxkit/moby/build.go index 484aa89d3..93d83336c 100644 --- a/src/cmd/linuxkit/moby/build.go +++ b/src/cmd/linuxkit/moby/build.go @@ -51,10 +51,11 @@ var additions = map[string]addFun{ "docker": func(tw *tar.Writer) error { log.Infof(" Adding Dockerfile") hdr := &tar.Header{ - Name: "Dockerfile", - Mode: 0644, - Size: int64(len(dockerfile)), - Format: tar.FormatPAX, + Name: "Dockerfile", + Mode: 0644, + Size: int64(len(dockerfile)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err @@ -368,6 +369,7 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { Name: "boot", Mode: 0755, Typeflag: tar.TypeDir, + ModTime: defaultModTime, Format: tar.FormatPAX, } if err := tw.WriteHeader(whdr); err != nil { @@ -376,10 +378,11 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { } // add the cmdline in /boot/cmdline whdr := &tar.Header{ - Name: "boot/cmdline", - Mode: 0644, - Size: int64(len(k.cmdline)), - Format: tar.FormatPAX, + Name: "boot/cmdline", + Mode: 0644, + Size: int64(len(k.cmdline)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(whdr); err != nil { return err @@ -391,10 +394,11 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { } // Stash the kernel header and prime the buffer for the kernel k.hdr = &tar.Header{ - Name: "boot/kernel", - Mode: hdr.Mode, - Size: hdr.Size, - Format: tar.FormatPAX, + Name: "boot/kernel", + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: defaultModTime, + Format: tar.FormatPAX, } k.buffer = new(bytes.Buffer) case k.tar: @@ -410,6 +414,7 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { Name: "boot", Mode: 0755, Typeflag: tar.TypeDir, + ModTime: defaultModTime, Format: tar.FormatPAX, } if err := tw.WriteHeader(whdr); err != nil { @@ -417,10 +422,11 @@ func (k *kernelFilter) WriteHeader(hdr *tar.Header) error { } } whdr := &tar.Header{ - Name: "boot/ucode.cpio", - Mode: hdr.Mode, - Size: hdr.Size, - Format: tar.FormatPAX, + Name: "boot/ucode.cpio", + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(whdr); err != nil { return err @@ -653,6 +659,7 @@ func filesystem(m Moby, tw *tar.Writer, idMap map[string]uint32) error { Name: root, Typeflag: tar.TypeDir, Mode: dirMode, + ModTime: defaultModTime, Uid: int(uid), Gid: int(gid), Format: tar.FormatPAX, @@ -666,11 +673,12 @@ func filesystem(m Moby, tw *tar.Writer, idMap map[string]uint32) error { } addedFiles[f.Path] = true hdr := &tar.Header{ - Name: f.Path, - Mode: mode, - Uid: int(uid), - Gid: int(gid), - Format: tar.FormatPAX, + Name: f.Path, + Mode: mode, + ModTime: defaultModTime, + Uid: int(uid), + Gid: int(gid), + Format: tar.FormatPAX, } if f.Directory { if f.Contents != nil { diff --git a/src/cmd/linuxkit/moby/config.go b/src/cmd/linuxkit/moby/config.go index d41677034..f4054a3ff 100644 --- a/src/cmd/linuxkit/moby/config.go +++ b/src/cmd/linuxkit/moby/config.go @@ -2,8 +2,6 @@ package moby import ( "fmt" - "os" - "path/filepath" "sort" "strconv" "strings" @@ -416,22 +414,6 @@ func defaultMountpoint(tp string) string { } } -// Sort mounts by number of path components so /dev/pts is listed after /dev -type mlist []specs.Mount - -func (m mlist) Len() int { - return len(m) -} -func (m mlist) Less(i, j int) bool { - return m.parts(i) < m.parts(j) -} -func (m mlist) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} -func (m mlist) parts(i int) int { - return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator)) -} - // assignBool does ordered overrides from JSON bool pointers func assignBool(v1, v2 *bool) bool { if v2 != nil { @@ -821,11 +803,15 @@ func ConfigInspectToOCI(yaml *Image, inspect types.ImageInspect, idMap map[strin } mounts[dest] = specs.Mount{Destination: dest, Type: tp, Source: src, Options: opts} } - mountList := mlist{} + // Create a list of mounts and sort them by destination. This makes the order deterministic and + // ensures that, e.g., the mount for /dev comes before the mount for /dev/pts + var mountList []specs.Mount for _, m := range mounts { mountList = append(mountList, m) } - sort.Sort(mountList) + sort.Slice(mountList, func(i, j int) bool { + return mountList[i].Destination < mountList[j].Destination + }) namespaces := []specs.LinuxNamespace{} @@ -915,6 +901,8 @@ func ConfigInspectToOCI(yaml *Image, inspect types.ImageInspect, idMap map[strin for capability := range boundingSet { bounding = append(bounding, capability) } + // Sort capabilities to make it deterministic + sort.Strings(bounding) rlimitsString := assignStrings(label.Rlimits, yaml.Rlimits) rlimits := []specs.POSIXRlimit{} diff --git a/src/cmd/linuxkit/moby/image.go b/src/cmd/linuxkit/moby/image.go index ef16416a1..624e30b34 100644 --- a/src/cmd/linuxkit/moby/image.go +++ b/src/cmd/linuxkit/moby/image.go @@ -53,6 +53,18 @@ ff02::2 ip6-allrouters `, } +// Files which may be created as part of 'docker export'. These need their timestamp fixed. +var touch = map[string]bool{ + "dev/": true, + "dev/pts/": true, + "dev/shm/": true, + "etc/": true, + "etc/mtab": true, + "etc/resolv.conf": true, + "proc/": true, + "sys/": true, +} + // tarPrefix creates the leading directories for a path func tarPrefix(path string, tw tarWriter) error { if path == "" { @@ -71,6 +83,7 @@ func tarPrefix(path string, tw tarWriter) error { hdr := &tar.Header{ Name: mkdir, Mode: 0755, + ModTime: defaultModTime, Typeflag: tar.TypeDir, Format: tar.FormatPAX, } @@ -154,7 +167,8 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull contents := replace[hdr.Name] hdr.Size = int64(len(contents)) hdr.Name = prefix + hdr.Name - log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name) + hdr.ModTime = defaultModTime + log.Debugf("image tar: %s %s add %s (replaced)", ref, prefix, hdr.Name) if err := tw.WriteHeader(hdr); err != nil { return err } @@ -169,6 +183,7 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull hdr.Size = 0 hdr.Typeflag = tar.TypeSymlink hdr.Linkname = resolv + hdr.ModTime = defaultModTime log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv) if err := tw.WriteHeader(hdr); err != nil { return err @@ -179,7 +194,12 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull return err } } else { - log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name) + if touch[hdr.Name] { + log.Debugf("image tar: %s %s add %s (touch)", ref, prefix, hdr.Name) + hdr.ModTime = defaultModTime + } else { + log.Debugf("image tar: %s %s add %s (original)", ref, prefix, hdr.Name) + } hdr.Name = prefix + hdr.Name if hdr.Typeflag == tar.TypeLink { // hard links are referenced by full path so need to be adjusted @@ -221,10 +241,11 @@ func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runt } hdr := &tar.Header{ - Name: path.Join(prefix, "config.json"), - Mode: 0644, - Size: int64(len(config)), - Format: tar.FormatPAX, + Name: path.Join(prefix, "config.json"), + Mode: 0644, + Size: int64(len(config)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err @@ -242,6 +263,7 @@ func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runt Name: tmp, Mode: 0755, Typeflag: tar.TypeDir, + ModTime: defaultModTime, Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { @@ -252,6 +274,7 @@ func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runt Name: path.Join(prefix, "rootfs"), Mode: 0755, Typeflag: tar.TypeDir, + ModTime: defaultModTime, Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { @@ -271,6 +294,7 @@ func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runt Name: path.Join(prefix, "rootfs"), Mode: 0755, Typeflag: tar.TypeDir, + ModTime: defaultModTime, Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { @@ -294,10 +318,11 @@ func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runt } hdr = &tar.Header{ - Name: path.Join(prefix, "runtime.json"), - Mode: 0644, - Size: int64(len(runtimeConfig)), - Format: tar.FormatPAX, + Name: path.Join(prefix, "runtime.json"), + Mode: 0644, + Size: int64(len(runtimeConfig)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err diff --git a/src/cmd/linuxkit/moby/output.go b/src/cmd/linuxkit/moby/output.go index 4b51e3ab1..dc8f90e01 100644 --- a/src/cmd/linuxkit/moby/output.go +++ b/src/cmd/linuxkit/moby/output.go @@ -281,10 +281,11 @@ func tarInitrdKernel(kernel, initrd []byte, cmdline string) (*bytes.Buffer, erro buf := new(bytes.Buffer) tw := tar.NewWriter(buf) hdr := &tar.Header{ - Name: "kernel", - Mode: 0600, - Size: int64(len(kernel)), - Format: tar.FormatPAX, + Name: "kernel", + Mode: 0600, + Size: int64(len(kernel)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } err := tw.WriteHeader(hdr) if err != nil { @@ -295,10 +296,11 @@ func tarInitrdKernel(kernel, initrd []byte, cmdline string) (*bytes.Buffer, erro return buf, err } hdr = &tar.Header{ - Name: "initrd.img", - Mode: 0600, - Size: int64(len(initrd)), - Format: tar.FormatPAX, + Name: "initrd.img", + Mode: 0600, + Size: int64(len(initrd)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } err = tw.WriteHeader(hdr) if err != nil { @@ -309,10 +311,11 @@ func tarInitrdKernel(kernel, initrd []byte, cmdline string) (*bytes.Buffer, erro return buf, err } hdr = &tar.Header{ - Name: "cmdline", - Mode: 0600, - Size: int64(len(cmdline)), - Format: tar.FormatPAX, + Name: "cmdline", + Mode: 0600, + Size: int64(len(cmdline)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } err = tw.WriteHeader(hdr) if err != nil { @@ -410,10 +413,11 @@ func outputKernelInitrdTarball(base string, kernel []byte, initrd []byte, cmdlin tw := tar.NewWriter(f) if len(kernel) != 0 { hdr := &tar.Header{ - Name: "kernel", - Mode: 0644, - Size: int64(len(kernel)), - Format: tar.FormatPAX, + Name: "kernel", + Mode: 0644, + Size: int64(len(kernel)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err @@ -424,10 +428,11 @@ func outputKernelInitrdTarball(base string, kernel []byte, initrd []byte, cmdlin } if len(initrd) != 0 { hdr := &tar.Header{ - Name: "initrd.img", - Mode: 0644, - Size: int64(len(initrd)), - Format: tar.FormatPAX, + Name: "initrd.img", + Mode: 0644, + Size: int64(len(initrd)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err @@ -438,10 +443,11 @@ func outputKernelInitrdTarball(base string, kernel []byte, initrd []byte, cmdlin } if len(cmdline) != 0 { hdr := &tar.Header{ - Name: "cmdline", - Mode: 0644, - Size: int64(len(cmdline)), - Format: tar.FormatPAX, + Name: "cmdline", + Mode: 0644, + Size: int64(len(cmdline)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err @@ -452,10 +458,11 @@ func outputKernelInitrdTarball(base string, kernel []byte, initrd []byte, cmdlin } if len(ucode) != 0 { hdr := &tar.Header{ - Name: "ucode.cpio", - Mode: 0644, - Size: int64(len(ucode)), - Format: tar.FormatPAX, + Name: "ucode.cpio", + Mode: 0644, + Size: int64(len(ucode)), + ModTime: defaultModTime, + Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err diff --git a/src/cmd/linuxkit/moby/util.go b/src/cmd/linuxkit/moby/util.go index 56afefb88..8765f87f6 100644 --- a/src/cmd/linuxkit/moby/util.go +++ b/src/cmd/linuxkit/moby/util.go @@ -2,11 +2,14 @@ package moby import ( "path/filepath" + "time" ) var ( // MobyDir is the location of the cache directory, defaults to ~/.moby MobyDir string + // Default ModTime for files created during build. Roughly the time LinuxKit got open sourced. + defaultModTime = time.Date(2017, time.April, 18, 16, 30, 0, 0, time.UTC) ) func defaultMobyConfigDir() string { diff --git a/test/cases/000_build/010_reproducible/000_tar/test.sh b/test/cases/000_build/010_reproducible/000_tar/test.sh new file mode 100644 index 000000000..cfd8c099b --- /dev/null +++ b/test/cases/000_build/010_reproducible/000_tar/test.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# SUMMARY: Check that tar output format build is reproducible +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +NAME=check + +clean_up() { + rm -f ${NAME}* +} + +trap clean_up EXIT + +# -disable-content-trust to speed up the test +linuxkit build -disable-content-trust -format tar -name "${NAME}-1" ../test.yml +linuxkit build -disable-content-trust -format tar -name "${NAME}-2" ../test.yml + +diff -q "${NAME}-1.tar" "${NAME}-2.tar" || exit 1 + +exit 0 diff --git a/test/cases/000_build/010_reproducible/002_kernel+initrd/test.sh b/test/cases/000_build/010_reproducible/002_kernel+initrd/test.sh new file mode 100644 index 000000000..260cb7dad --- /dev/null +++ b/test/cases/000_build/010_reproducible/002_kernel+initrd/test.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# SUMMARY: Check that kernel+initrd output format build is reproducible +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +NAME=check + +clean_up() { + rm -f ${NAME}* +} + +trap clean_up EXIT + +# -disable-content-trust to speed up the test +linuxkit build -disable-content-trust -format kernel+initrd -name "${NAME}-1" ../test.yml +linuxkit build -disable-content-trust -format kernel+initrd -name "${NAME}-2" ../test.yml + +diff -q "${NAME}-1-cmdline" "${NAME}-2-cmdline" || exit 1 +diff -q "${NAME}-1-kernel" "${NAME}-2-kernel" || exit 1 +diff -q "${NAME}-1-initrd.img" "${NAME}-2-initrd.img" || exit 1 + +exit 0 diff --git a/test/cases/000_build/010_reproducible/test.yml b/test/cases/000_build/010_reproducible/test.yml new file mode 100644 index 000000000..a4934cafd --- /dev/null +++ b/test/cases/000_build/010_reproducible/test.yml @@ -0,0 +1,52 @@ +# NOTE: Images build from this file likely do not run +kernel: + image: linuxkit/kernel:4.14.90 + cmdline: "console=ttyS0" +init: + - linuxkit/init:c563953a2277eb73a89d89f70e4b6dcdcfebc2d1 + - linuxkit/runc:83d0edb4552b1a5df1f0976f05f442829eac38fe + - linuxkit/containerd:326b096cd5fbab0f864e52721d036cade67599d6 + +onboot: + - name: dhcpcd + image: linuxkit/dhcpcd:v0.6 + command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] + # Add some random unsorted caps + capabilities: + - CAP_SETGID + - CAP_DAC_OVERRIDE + +services: + - name: testservice + image: linuxkit/ip:v0.6 + # Some environments + env: + - BENV=true + - ARANDOMENV=foobar + # Some mounts + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + - type: overlay + source: overlay + destination: writeable-host-etc + options: ["rw", "lowerdir=/etc", "upperdir=/run/hostetc/upper", "workdir=/run/hostetc/work"] + # Some binds + binds: + - /var/run:/var/run + - /foobar:/foobar + - /etc/foobar:/etc/foobar + - /etc/aaa:/etc/aaa + # And some runtime settings + runtime: + mkdir: ["/var/lib/docker"] + mkdir: ["/var/lib/aaa"] + +files: + - path: etc/linuxkit-config + metadata: yaml + +trust: + org: + - linuxkit + - library