Merge pull request #2703 from guillaumerose/multi

metadata: handle json with more than 2 levels

blueprints/docker-for-mac/base.yml

@@ -11,7 +11,7 @@ init:
 onboot:
   # support metadata for optional config in /var/config
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
   - name: sysctl
     image: linuxkit/sysctl:a9ad57ed738a31ea9380cd73236866c312b35489
   - name: sysfs

blueprints/docker-for-mac/metadata.json

@@ -1,8 +1,10 @@
 {
   "docker": {
-    "daemon.json": {
+    "entries": {
-      "perm": "0644",
+      "daemon.json": {
-      "content": "{ \"debug\": true }"
+        "perm": "0644",
+        "content": "{ \"debug\": true }"
+      }
     }
   }
-}
+}

docs/metadata.md

@@ -22,19 +22,25 @@ directories to be created and the directories are populated with files.
 For example, the following userdata file:
 ```JSON
 {
-    "ssh" : {
+  "ssh": {
-        "sshd_config" : {
+    "entries": {
-            "perm" : "0600",
+      "sshd_config": {
-            "content": "PermitRootLogin yes\nPasswordAuthentication no"
+        "perm": "0600",
-        }
+        "content": "PermitRootLogin yes\nPasswordAuthentication no"
-    },
+      }
-    "foo" : {
-        "bar" : "foobar",
-        "baz" : {
-            "perm": "0600",
-            "content": "bar"
-        }
     }
+  },
+  "foo": {
+    "entries": {
+      "bar": {
+        "content": "foobar"
+      },
+      "baz": {
+        "perm": "0600",
+        "content": "bar"
+      }
+    }
+  }
 }
 ```
 will generate the following files:
@@ -44,16 +50,15 @@ will generate the following files:
 /var/config/foo/baz
 ```
 
-Each file can either be:
+The JSON file consists of a map from `name` to an entry object. Each entry object has the following fields:
-
+- `content`: if present then the entry is a file. The value is a string containing the desired contents of the file.
-- a simple string (as for `foo/bar` above) in which case the file will
+- `entries`: if present then the entry is a directory. The value is a map from `name` to entry objects.
-  be created with the given contents and read/write (but not execute)
+- `perm`: the permissions to create the file with.
-  permissions for user and read permissions for group and everyone else (in octal format `0644`).
-- a map (as for `ssh/sshd_config` and `foo/baz` above) with the
-  following mandatory keys:
-  - `content`: the contents of the file.
-  - `perm`: the permissions to create the file with.
 
+The `content` and `entries` fields are mutually exclusive, it is an error to include both, 
+one or the other _must_ be present. 
+The file or directory's name in each case is the same as the key which referred to that entry.
+ 
 This hierarchy can then be used by individual containers, who can bind
 mount the config sub-directory into their namespace where it is
 needed.

examples/aws.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: linuxkit/rngd:842e5e8ece7934f0cab9fd0027b595ff3471e5b9

examples/gcp.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa

examples/openstack.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
     command: ["/usr/bin/metadata", "openstack"]
 services:
   - name: rngd

examples/packet.yml

@@ -16,7 +16,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
     command: ["/usr/bin/metadata", "packet"]
 services:
   - name: rngd

examples/vultr.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa

pkg/metadata/main.go

@@ -111,7 +111,7 @@ func main() {
 	}
 
 	if userdata != nil {
-		if err := processUserData(userdata); err != nil {
+		if err := processUserData(ConfigPath, userdata); err != nil {
 			log.Printf("Could not extract user data: %s", err)
 		}
 	}
@@ -139,70 +139,82 @@ func main() {
 //        }
 // }
 // Will create foobar/foo with mode 0644 and content "hello"
-func processUserData(data []byte) error {
+func processUserData(basePath string, data []byte) error {
 	// Always write the raw data to a file
-	err := ioutil.WriteFile(path.Join(ConfigPath, "userdata"), data, 0644)
+	err := ioutil.WriteFile(path.Join(basePath, "userdata"), data, 0644)
 	if err != nil {
 		log.Printf("Could not write userdata: %s", err)
 		return err
 	}
 
-	var fd interface{}
+	var root ConfigFile
-	if err := json.Unmarshal(data, &fd); err != nil {
+	if err := json.Unmarshal(data, &root); err != nil {
 		// Userdata is no JSON, presumably...
 		log.Printf("Could not unmarshall userdata: %s", err)
 		// This is not an error
 		return nil
 	}
-	cm, ok := fd.(map[string]interface{})
+
-	if !ok {
+	for dir, entry := range root {
-		log.Printf("Could convert JSON to desired format: %s", fd)
+		writeConfigFiles(path.Join(basePath, dir), entry)
-		return nil
 	}
-	for d, val := range cm {
-		dir := path.Join(ConfigPath, d)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			log.Printf("Failed to create %s: %s", dir, err)
-			continue
-		}
-		files, ok := val.(map[string]interface{})
-		if !ok {
-			log.Printf("Could convert JSON for files: %s", val)
-			continue
-		}
-		for f, i := range files {
-			p := uint64(0644)
-			var c string
-
-			switch fi := i.(type) {
-			case map[string]interface{}:
-				if _, ok := fi["perm"]; !ok {
-					log.Printf("No permission provided %s", f)
-					continue
-				}
-				if _, ok := fi["content"]; !ok {
-					log.Printf("No content provided %s", f)
-					continue
-				}
-				c = fi["content"].(string)
-				if p, err = strconv.ParseUint(fi["perm"].(string), 8, 32); err != nil {
-					log.Printf("Failed to parse permission %s: %s", fi, err)
-					continue
-				}
-			case string:
-				c = fi
-			default:
-				log.Printf("Couldn't convert JSON for items: %s", i)
-				continue
-			}
-
-			if err := ioutil.WriteFile(path.Join(dir, f), []byte(c), os.FileMode(p)); err != nil {
-				log.Printf("Failed to write %s/%s: %s", dir, f, err)
-				continue
-
-			}
-		}
-	}
-
 	return nil
 }
+
+func writeConfigFiles(target string, current Entry) {
+	if isFile(current) {
+		filemode, err := parseFileMode(current.Perm, 0644)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := ioutil.WriteFile(target, []byte(*current.Content), filemode); err != nil {
+			log.Printf("Failed to write %s: %s", target, err)
+			return
+		}
+	} else if isDirectory(current) {
+		filemode, err := parseFileMode(current.Perm, 0755)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := os.MkdirAll(target, filemode); err != nil {
+			log.Printf("Failed to create %s: %s", target, err)
+			return
+		}
+		for dir, entry := range current.Entries {
+			writeConfigFiles(path.Join(target, dir), entry)
+		}
+	} else {
+		log.Printf("%s is invalid", target)
+	}
+}
+
+func isFile(json Entry) bool {
+	return json.Content != nil && json.Entries == nil
+}
+
+func isDirectory(json Entry) bool {
+	return json.Content == nil && json.Entries != nil
+}
+
+func parseFileMode(input string, defaultMode os.FileMode) (os.FileMode, error) {
+	if input != "" {
+		perm, err := strconv.ParseUint(input, 8, 32)
+		if err != nil {
+			return 0, err
+		}
+		return os.FileMode(perm), nil
+	}
+	return defaultMode, nil
+}
+
+// ConfigFile represents the configuration file
+type ConfigFile map[string]Entry
+
+// Entry represents either a directory or a file
+type Entry struct {
+	Perm    string           `json:"perm,omitempty"`
+	Content *string          `json:"content,omitempty"`
+	Entries map[string]Entry `json:"entries,omitempty"`
+}

pkg/metadata/main_test.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestSampleConfig(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "ssh": {
+		"entries": {
+		  "sshd_config": {
+			"perm": "0600",
+			"content": "PermitRootLogin yes\nPasswordAuthentication no"
+		  }
+		}
+	  },
+	  "foo": {
+		"entries": {
+		  "bar": {
+			"content": "foobar"
+		  },
+		  "baz": {
+			"perm": "0600",
+			"content": "bar"
+		  }
+		}
+	  }
+	}`)
+
+	sshd := path.Join(basePath, "ssh", "sshd_config")
+	assertContent(t, sshd, "PermitRootLogin yes\nPasswordAuthentication no")
+	assertPermission(t, sshd, 0600)
+
+	bar := path.Join(basePath, "foo", "bar")
+	assertContent(t, bar, "foobar")
+	assertPermission(t, bar, 0644)
+
+	assertContent(t, path.Join(basePath, "foo", "baz"), "bar")
+}
+
+func TestSerialization(t *testing.T) {
+	bin, err := json.Marshal(ConfigFile{
+		"ssh": Entry{
+			Entries: map[string]Entry{
+				"sshd_config": {
+					Content: str("PermitRootLogin yes\nPasswordAuthentication no"),
+					Perm:    "0600",
+				},
+			},
+		},
+		"foo": Entry{
+			Entries: map[string]Entry{
+				"bar": {
+					Content: str("foobar"),
+				},
+				"baz": {
+					Content: str("bar"),
+					Perm:    "0600",
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Cannot convert to json: %v", err)
+	}
+
+	expected := `{"foo":{"entries":{"bar":{"content":"foobar"},"baz":{"perm":"0600","content":"bar"}}},"ssh":{"entries":{"sshd_config":{"perm":"0600","content":"PermitRootLogin yes\nPasswordAuthentication no"}}}}`
+	if expected != string(bin) {
+		t.Fatalf("Expected %v but has %v", expected, string(bin))
+	}
+}
+
+func TestWriteSingleFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "hostname": {
+		"content": "foobar"
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "hostname"), "foobar")
+}
+
+func TestWriteEmptyFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"content": ""
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "empty"), "")
+}
+
+func TestWriteEmptyDirectory(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"entries": {}
+	  }
+	}`)
+
+	if _, err := os.Stat(path.Join(basePath, "empty")); err != nil {
+		t.Fatalf("empty folder doesn't exist: %v", err)
+	}
+}
+
+func TestSetPermission(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "restricted": {
+		"perm": "0600",
+		"entries": {
+		  "password": {
+			"perm": "0600",
+			"content": "secret"
+		  }
+		}
+	  }
+	}`)
+
+	assertPermission(t, path.Join(basePath, "restricted"), 0600|os.ModeDir)
+	assertPermission(t, path.Join(basePath, "restricted", "password"), 0600)
+}
+
+func TestDeepTree(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "level1": {
+		"entries": {
+		  "level2": {
+			"entries": {
+			  "file2": {
+				"content": "depth2"
+			  },
+			  "level3": {
+				"entries": {
+				  "file3": {
+					"content": "depth3"
+				  }
+				}
+			  }
+			}
+		  }
+		}
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "level1", "level2", "level3", "file3"), "depth3")
+	assertContent(t, path.Join(basePath, "level1", "level2", "file2"), "depth2")
+}
+
+func str(input string) *string {
+	return &input
+}
+
+func process(t *testing.T, basePath string, json string) {
+	if err := processUserData(basePath, []byte(json)); err != nil {
+		t.Fatalf("fail to process json %v", err)
+	}
+}
+
+func assertPermission(t *testing.T, path string, expected os.FileMode) {
+	fileinfo, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("%v doesn't exist: %v", path, err)
+	}
+	if fileinfo.Mode() != expected {
+		t.Fatalf("%v: expected %v but has %v", path, expected, fileinfo.Mode())
+	}
+}
+
+func assertContent(t *testing.T, path, expected string) {
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Fatalf("can't read %v: %v", path, err)
+	}
+	if !bytes.Equal(file, []byte(expected)) {
+		t.Fatalf("%v: expected %v but has %v", path, string(expected), string(file))
+	}
+}

projects/etcd/etcd.yml

@@ -18,7 +18,7 @@ onboot:
     image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: linuxkit/rngd:45ed7759dd927f4cce3863073ea2e0da1d52a427

projects/etcd/prom-us-central1-f.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9

projects/swarmd/swarmd.yml

@@ -20,7 +20,7 @@ onboot:
     image: linuxkit/mount:41685ecc8039643948e5dff46e17584753469a7a
     command: ["/usr/bin/mountie", "/var/lib/swarmd"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa