Merge pull request #3511 from deitch/append-binds

support merge yaml flags

docs/yaml.md

@@ -13,7 +13,7 @@ so it can be tested reliably for continuous delivery.
 Components are specified as Docker images which are pulled from a registry during build if they
 are not available locally. The Docker images are optionally verified with Docker Content Trust.
 For private registries or private repositories on a registry credentials provided via
-`docker login` are re-used. 
+`docker login` are re-used.
 
 The configuration file is processed in the order `kernel`, `init`, `onboot`, `onshutdown`,
 `services`, `files`. Each section adds files to the root file system. Sections may be omitted.
@@ -144,7 +144,9 @@ options. Default values may be specified using the `org.mobyproject.config` imag
 For more details see the [OCI specification](https://github.com/opencontainers/runtime-spec/blob/master/spec.md).
 
 If the `org.mobylinux.config` label is set in the image, that specifies default values for these fields if they
-are not set in the yaml file. You can override the label by setting the value, or setting it to be empty to remove
+are not set in the yaml file. While most fields are _replaced_ if they are specified in the yaml file,
+some support _add_ via the format `<field>.add`; see below.
+You can override the label entirely by setting the value, or setting it to be empty to remove
 the specification for that value in the label.
 
 If you need an OCI option that is not specified here please open an issue or pull request as the list is not yet
@@ -159,6 +161,7 @@ bind mounted into a container.
   extracted from this so they need not be filled in.
 - `capabilities` the Linux capabilities required, for example `CAP_SYS_ADMIN`. If there is a single
   capability `all` then all capabilities are added.
+- `capabilities.add` the Linux capabilities required, but these are added to the defaults, rather than overriding them.
 - `ambient` the Linux ambient capabilities (capabilities passed to non root users) that are required.
 - `mounts` is the full form for specifying a mount, which requires `type`, `source`, `destination`
   and a list of `options`. If any fields are omitted, sensible defaults are used if possible, for example
@@ -166,6 +169,7 @@ bind mounted into a container.
   can be replaced by specifying a mount with new options here at the same mount point.
 - `binds` is a simpler interface to specify bind mounts, accepting a string like `/src:/dest:opt1,opt2`
   similar to the `-v` option for bind mounts in Docker.
+- `binds.add` is a simpler interface to specify bind mounts, but these are added to the defaults, rather than overriding them.
 - `tmpfs` is a simpler interface to mount a `tmpfs`, like `--tmpfs` in Docker, taking `/dest:opt1,opt2`.
 - `command` will override the command and entrypoint in the image with a new list of commands.
 - `env` will override the environment in the image with a new environment list. Specify variables as `VAR=value`.

examples/addbinds.yml

@@ -0,0 +1,29 @@
+kernel:
+  image: linuxkit/kernel:5.4.30
+  cmdline: "console=tty0 console=ttyS0 console=ttyAMA0 console=ttysclp0"
+init:
+  - linuxkit/init:7195dc244cd92af01fd0895fd204249a6114c5e2
+  - linuxkit/runc:f79954950022fea76b8b6f10de58cb48e4fb3878
+  - linuxkit/containerd:8ee7a0d636fff9df7e13076f5492d06274e5f644
+  - linuxkit/ca-certificates:abfc6701b9ca17e34ac9439ce5946a247e720ff5
+onboot:
+  - name: sysctl
+    image: linuxkit/sysctl:541f60fe3676611328e89e8bac251fc636b1a6aa
+  - name: dhcpcd
+    image: linuxkit/dhcpcd:2f8a9b670aa6e96a09db56ec45c9f07ef2a811ee
+    command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
+services:
+  - name: getty
+    image: linuxkit/getty:48f66df198981e692084bf70ab72b9fe2be0f880
+    binds.add:
+      # this will keep all of the existing ones as well
+      - /var/tmp:/var/tmp
+  - name: rngd
+    image: linuxkit/rngd:7fab61cca793113280397dcee8159f35dc37adcb
+files:
+  - path: etc/getty.shadow
+    # sample sets password for root to "abcdefgh" (without quotes)
+    contents: 'root:$6$6tPd2uhHrecCEKug$8mKfcgfwguP7f.BLdZsT1Wz7WIIJOBY1oUFHzIv9/O71M2J0EPdtFqFGTxB1UK5ejqQxRFQ.ZSG9YXR0SNsc11:17322:0:::::'
+trust:
+  org:
+    - linuxkit

src/cmd/linuxkit/moby/config.go

@@ -71,9 +71,11 @@ type Image struct {
 // Everything except Runtime and ref is used to build the OCI spec
 type ImageConfig struct {
 	Capabilities      *[]string               `yaml:"capabilities,omitempty" json:"capabilities,omitempty"`
+	CapabilitiesAdd   *[]string               `yaml:"capabilities.add,omitempty" json:"capabilities.add,omitempty"`
 	Ambient           *[]string               `yaml:"ambient,omitempty" json:"ambient,omitempty"`
 	Mounts            *[]specs.Mount          `yaml:"mounts,omitempty" json:"mounts,omitempty"`
 	Binds             *[]string               `yaml:"binds,omitempty" json:"binds,omitempty"`
+	BindsAdd          *[]string               `yaml:"binds.add,omitempty" json:"binds.add,omitempty"`
 	Tmpfs             *[]string               `yaml:"tmpfs,omitempty" json:"tmpfs,omitempty"`
 	Command           *[]string               `yaml:"command,omitempty" json:"command,omitempty"`
 	Env               *[]string               `yaml:"env,omitempty" json:"env,omitempty"`
@@ -505,6 +507,34 @@ func assignStrings3(v1 []string, v2, v3 *[]string) []string {
 	return v1
 }
 
+// mergeStrings does ordered unique merge between JSON string array pointers
+func mergeStrings(v1, v2 *[]string) *[]string {
+	switch {
+	case v2 == nil && v1 == nil:
+		return &[]string{}
+	case v2 == nil:
+		return v1
+	case v1 == nil:
+		return v2
+	}
+	// merge the two uniquely
+	ret := []string{}
+	m := make(map[string]bool)
+	for _, s := range *v1 {
+		if m[s] {
+			continue
+		}
+		ret = append(ret, s)
+	}
+	for _, s := range *v2 {
+		if m[s] {
+			continue
+		}
+		ret = append(ret, s)
+	}
+	return &ret
+}
+
 // assignMaps does ordered overrides from JSON string map pointers
 func assignMaps(v1, v2 *map[string]string) map[string]string {
 	if v2 != nil {
@@ -770,7 +800,7 @@ func ConfigInspectToOCI(yaml *Image, inspect types.ImageInspect, idMap map[strin
 		}
 		mounts[dest] = specs.Mount{Destination: dest, Type: "tmpfs", Source: "tmpfs", Options: opts}
 	}
-	for _, b := range assignStrings(label.Binds, yaml.Binds) {
+	for _, b := range assignStrings(mergeStrings(label.Binds, yaml.BindsAdd), yaml.Binds) {
 		parts := strings.Split(b, ":")
 		if len(parts) < 2 {
 			return oci, runtime, fmt.Errorf("Cannot parse bind, missing ':': %s", b)
@@ -880,7 +910,7 @@ func ConfigInspectToOCI(yaml *Image, inspect types.ImageInspect, idMap map[strin
 		capCheck[capability] = true
 	}
 	boundingSet := map[string]bool{}
-	caps := assignStrings(label.Capabilities, yaml.Capabilities)
+	caps := assignStrings(mergeStrings(label.Capabilities, yaml.CapabilitiesAdd), yaml.Capabilities)
 	if len(caps) == 1 {
 		switch cap := strings.ToLower(caps[0]); cap {
 		case "none":

src/cmd/linuxkit/moby/schema.go

@@ -260,9 +260,11 @@ var schema = string(`
         "name": {"type": "string"},
         "image": {"type": "string"},
         "capabilities": { "$ref": "#/definitions/strings" },
+        "capabilities.add": { "$ref": "#/definitions/strings" },
         "ambient": { "$ref": "#/definitions/strings" },
         "mounts": { "$ref": "#/definitions/mounts" },
         "binds": { "$ref": "#/definitions/strings" },
+        "binds.add": { "$ref": "#/definitions/strings" },
         "tmpfs": { "$ref": "#/definitions/strings" },
         "command": { "$ref": "#/definitions/strings" },
         "env": { "$ref": "#/definitions/strings" },

test/cases/000_build/020_binds/check.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -x
+
+function failed {
+	printf "bindmerge test suite FAILED\n" >&1
+	exit 1
+}
+
+# the very fact that this is running means that the bind worked, so just need to check that the defaults also
+# are there
+
+[ -d /dev/mapper ] || failed
+[ -d /hostroot ] || failed
+printf "bindmerge test suite PASSED\n" >&1

test/cases/000_build/020_binds/test.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# SUMMARY: Check that the userdata is found and read when on a cidata partition
+# LABELS:
+# REPEAT:
+
+set -ex
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+NAME=bindtest
+DISK=disk.img
+
+clean_up() {
+	rm -rf ${NAME}-* ${DISK}
+}
+trap clean_up EXIT
+
+# Test code goes here
+
+linuxkit build -format kernel+initrd -name ${NAME} test.yml
+RESULT="$(linuxkit run -disk file=${DISK},size=32M ${NAME})"
+echo "${RESULT}"
+echo "${RESULT}" | grep -q "suite PASSED"
+
+exit 0

test/cases/000_build/020_binds/test.yml

@@ -0,0 +1,28 @@
+kernel:
+  image: linuxkit/kernel:5.4.30
+  cmdline: "console=ttyS0 console=ttyAMA0"
+init:
+  - linuxkit/init:7195dc244cd92af01fd0895fd204249a6114c5e2
+  - linuxkit/runc:f79954950022fea76b8b6f10de58cb48e4fb3878
+onboot:
+  - name: mount
+    image: linuxkit/mount:19fa297189166206ac97261679c3e31fb140d48f
+    binds.add:
+      - /check.sh:/check.sh
+      - /var/tmp:/var/tmp
+    # default binds from linuxkit/mount
+    # - /dev:/dev
+    # - /var:/var:rshared,rbind
+    # - /:/hostroot
+    command: ["sh", "-c", "/check.sh"]
+  - name: poweroff
+    image: linuxkit/poweroff:06dd4e46c62fbe79123a028835c921f80e4855d3
+    command: ["/bin/sh", "/poweroff.sh", "10"]
+files:
+  - path: check.sh
+    source: ./check.sh
+    mode: "0700"
+trust:
+  org:
+    - linuxkit
+    - library