Merge pull request #2703 from guillaumerose/multi

metadata: handle json with more than 2 levels

blueprints/docker-for-mac/base.yml

@@ -11,7 +11,7 @@ init:
 onboot:
   # support metadata for optional config in /var/config
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
   - name: sysctl
     image: linuxkit/sysctl:a9ad57ed738a31ea9380cd73236866c312b35489
   - name: sysfs

blueprints/docker-for-mac/metadata.json

@@ -1,8 +1,10 @@
 {
   "docker": {
+    "entries": {
       "daemon.json": {
         "perm": "0644",
         "content": "{ \"debug\": true }"
       }
     }
+  }
 }

docs/metadata.md

@@ -22,19 +22,25 @@ directories to be created and the directories are populated with files.
 For example, the following userdata file:
 ```JSON
 {
-    "ssh" : {
-        "sshd_config" : {
-            "perm" : "0600",
+  "ssh": {
+    "entries": {
+      "sshd_config": {
+        "perm": "0600",
         "content": "PermitRootLogin yes\nPasswordAuthentication no"
       }
+    }
   },
-    "foo" : {
-        "bar" : "foobar",
-        "baz" : {
+  "foo": {
+    "entries": {
+      "bar": {
+        "content": "foobar"
+      },
+      "baz": {
         "perm": "0600",
         "content": "bar"
       }
     }
+  }
 }
 ```
 will generate the following files:
@@ -44,15 +50,14 @@ will generate the following files:
 /var/config/foo/baz
 ```
 
-Each file can either be:
+The JSON file consists of a map from `name` to an entry object. Each entry object has the following fields:
+- `content`: if present then the entry is a file. The value is a string containing the desired contents of the file.
+- `entries`: if present then the entry is a directory. The value is a map from `name` to entry objects.
+- `perm`: the permissions to create the file with.
 
-- a simple string (as for `foo/bar` above) in which case the file will
-  be created with the given contents and read/write (but not execute)
-  permissions for user and read permissions for group and everyone else (in octal format `0644`).
-- a map (as for `ssh/sshd_config` and `foo/baz` above) with the
-  following mandatory keys:
-  - `content`: the contents of the file.
-  - `perm`: the permissions to create the file with.
+The `content` and `entries` fields are mutually exclusive, it is an error to include both, 
+one or the other _must_ be present. 
+The file or directory's name in each case is the same as the key which referred to that entry.
  
 This hierarchy can then be used by individual containers, who can bind
 mount the config sub-directory into their namespace where it is

examples/aws.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: linuxkit/rngd:842e5e8ece7934f0cab9fd0027b595ff3471e5b9

examples/gcp.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa

examples/openstack.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
     command: ["/usr/bin/metadata", "openstack"]
 services:
   - name: rngd

examples/packet.yml

@@ -16,7 +16,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
     command: ["/usr/bin/metadata", "packet"]
 services:
   - name: rngd

examples/vultr.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa

pkg/metadata/main.go

@@ -111,7 +111,7 @@ func main() {
 	}
 
 	if userdata != nil {
-		if err := processUserData(userdata); err != nil {
+		if err := processUserData(ConfigPath, userdata); err != nil {
 			log.Printf("Could not extract user data: %s", err)
 		}
 	}
@@ -139,70 +139,82 @@ func main() {
 //        }
 // }
 // Will create foobar/foo with mode 0644 and content "hello"
-func processUserData(data []byte) error {
+func processUserData(basePath string, data []byte) error {
 	// Always write the raw data to a file
-	err := ioutil.WriteFile(path.Join(ConfigPath, "userdata"), data, 0644)
+	err := ioutil.WriteFile(path.Join(basePath, "userdata"), data, 0644)
 	if err != nil {
 		log.Printf("Could not write userdata: %s", err)
 		return err
 	}
 
-	var fd interface{}
-	if err := json.Unmarshal(data, &fd); err != nil {
+	var root ConfigFile
+	if err := json.Unmarshal(data, &root); err != nil {
 		// Userdata is no JSON, presumably...
 		log.Printf("Could not unmarshall userdata: %s", err)
 		// This is not an error
 		return nil
 	}
-	cm, ok := fd.(map[string]interface{})
-	if !ok {
-		log.Printf("Could convert JSON to desired format: %s", fd)
-		return nil
-	}
-	for d, val := range cm {
-		dir := path.Join(ConfigPath, d)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			log.Printf("Failed to create %s: %s", dir, err)
-			continue
-		}
-		files, ok := val.(map[string]interface{})
-		if !ok {
-			log.Printf("Could convert JSON for files: %s", val)
-			continue
-		}
-		for f, i := range files {
-			p := uint64(0644)
-			var c string
 
-			switch fi := i.(type) {
-			case map[string]interface{}:
-				if _, ok := fi["perm"]; !ok {
-					log.Printf("No permission provided %s", f)
-					continue
+	for dir, entry := range root {
+		writeConfigFiles(path.Join(basePath, dir), entry)
 	}
-				if _, ok := fi["content"]; !ok {
-					log.Printf("No content provided %s", f)
-					continue
-				}
-				c = fi["content"].(string)
-				if p, err = strconv.ParseUint(fi["perm"].(string), 8, 32); err != nil {
-					log.Printf("Failed to parse permission %s: %s", fi, err)
-					continue
-				}
-			case string:
-				c = fi
-			default:
-				log.Printf("Couldn't convert JSON for items: %s", i)
-				continue
-			}
-
-			if err := ioutil.WriteFile(path.Join(dir, f), []byte(c), os.FileMode(p)); err != nil {
-				log.Printf("Failed to write %s/%s: %s", dir, f, err)
-				continue
-
-			}
-		}
-	}
-
 	return nil
 }
+
+func writeConfigFiles(target string, current Entry) {
+	if isFile(current) {
+		filemode, err := parseFileMode(current.Perm, 0644)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := ioutil.WriteFile(target, []byte(*current.Content), filemode); err != nil {
+			log.Printf("Failed to write %s: %s", target, err)
+			return
+		}
+	} else if isDirectory(current) {
+		filemode, err := parseFileMode(current.Perm, 0755)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := os.MkdirAll(target, filemode); err != nil {
+			log.Printf("Failed to create %s: %s", target, err)
+			return
+		}
+		for dir, entry := range current.Entries {
+			writeConfigFiles(path.Join(target, dir), entry)
+		}
+	} else {
+		log.Printf("%s is invalid", target)
+	}
+}
+
+func isFile(json Entry) bool {
+	return json.Content != nil && json.Entries == nil
+}
+
+func isDirectory(json Entry) bool {
+	return json.Content == nil && json.Entries != nil
+}
+
+func parseFileMode(input string, defaultMode os.FileMode) (os.FileMode, error) {
+	if input != "" {
+		perm, err := strconv.ParseUint(input, 8, 32)
+		if err != nil {
+			return 0, err
+		}
+		return os.FileMode(perm), nil
+	}
+	return defaultMode, nil
+}
+
+// ConfigFile represents the configuration file
+type ConfigFile map[string]Entry
+
+// Entry represents either a directory or a file
+type Entry struct {
+	Perm    string           `json:"perm,omitempty"`
+	Content *string          `json:"content,omitempty"`
+	Entries map[string]Entry `json:"entries,omitempty"`
+}

pkg/metadata/main_test.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestSampleConfig(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "ssh": {
+		"entries": {
+		  "sshd_config": {
+			"perm": "0600",
+			"content": "PermitRootLogin yes\nPasswordAuthentication no"
+		  }
+		}
+	  },
+	  "foo": {
+		"entries": {
+		  "bar": {
+			"content": "foobar"
+		  },
+		  "baz": {
+			"perm": "0600",
+			"content": "bar"
+		  }
+		}
+	  }
+	}`)
+
+	sshd := path.Join(basePath, "ssh", "sshd_config")
+	assertContent(t, sshd, "PermitRootLogin yes\nPasswordAuthentication no")
+	assertPermission(t, sshd, 0600)
+
+	bar := path.Join(basePath, "foo", "bar")
+	assertContent(t, bar, "foobar")
+	assertPermission(t, bar, 0644)
+
+	assertContent(t, path.Join(basePath, "foo", "baz"), "bar")
+}
+
+func TestSerialization(t *testing.T) {
+	bin, err := json.Marshal(ConfigFile{
+		"ssh": Entry{
+			Entries: map[string]Entry{
+				"sshd_config": {
+					Content: str("PermitRootLogin yes\nPasswordAuthentication no"),
+					Perm:    "0600",
+				},
+			},
+		},
+		"foo": Entry{
+			Entries: map[string]Entry{
+				"bar": {
+					Content: str("foobar"),
+				},
+				"baz": {
+					Content: str("bar"),
+					Perm:    "0600",
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Cannot convert to json: %v", err)
+	}
+
+	expected := `{"foo":{"entries":{"bar":{"content":"foobar"},"baz":{"perm":"0600","content":"bar"}}},"ssh":{"entries":{"sshd_config":{"perm":"0600","content":"PermitRootLogin yes\nPasswordAuthentication no"}}}}`
+	if expected != string(bin) {
+		t.Fatalf("Expected %v but has %v", expected, string(bin))
+	}
+}
+
+func TestWriteSingleFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "hostname": {
+		"content": "foobar"
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "hostname"), "foobar")
+}
+
+func TestWriteEmptyFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"content": ""
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "empty"), "")
+}
+
+func TestWriteEmptyDirectory(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"entries": {}
+	  }
+	}`)
+
+	if _, err := os.Stat(path.Join(basePath, "empty")); err != nil {
+		t.Fatalf("empty folder doesn't exist: %v", err)
+	}
+}
+
+func TestSetPermission(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "restricted": {
+		"perm": "0600",
+		"entries": {
+		  "password": {
+			"perm": "0600",
+			"content": "secret"
+		  }
+		}
+	  }
+	}`)
+
+	assertPermission(t, path.Join(basePath, "restricted"), 0600|os.ModeDir)
+	assertPermission(t, path.Join(basePath, "restricted", "password"), 0600)
+}
+
+func TestDeepTree(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "level1": {
+		"entries": {
+		  "level2": {
+			"entries": {
+			  "file2": {
+				"content": "depth2"
+			  },
+			  "level3": {
+				"entries": {
+				  "file3": {
+					"content": "depth3"
+				  }
+				}
+			  }
+			}
+		  }
+		}
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "level1", "level2", "level3", "file3"), "depth3")
+	assertContent(t, path.Join(basePath, "level1", "level2", "file2"), "depth2")
+}
+
+func str(input string) *string {
+	return &input
+}
+
+func process(t *testing.T, basePath string, json string) {
+	if err := processUserData(basePath, []byte(json)); err != nil {
+		t.Fatalf("fail to process json %v", err)
+	}
+}
+
+func assertPermission(t *testing.T, path string, expected os.FileMode) {
+	fileinfo, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("%v doesn't exist: %v", path, err)
+	}
+	if fileinfo.Mode() != expected {
+		t.Fatalf("%v: expected %v but has %v", path, expected, fileinfo.Mode())
+	}
+}
+
+func assertContent(t *testing.T, path, expected string) {
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Fatalf("can't read %v: %v", path, err)
+	}
+	if !bytes.Equal(file, []byte(expected)) {
+		t.Fatalf("%v: expected %v but has %v", path, string(expected), string(file))
+	}
+}

projects/etcd/etcd.yml

@@ -18,7 +18,7 @@ onboot:
     image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: linuxkit/rngd:45ed7759dd927f4cce3863073ea2e0da1d52a427

projects/etcd/prom-us-central1-f.yml

@@ -13,7 +13,7 @@ onboot:
     image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2
     command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
   - name: metadata
-    image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: rngd
     image: mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9

projects/swarmd/swarmd.yml

@@ -20,7 +20,7 @@ onboot:
     image: linuxkit/mount:41685ecc8039643948e5dff46e17584753469a7a
     command: ["/usr/bin/mountie", "/var/lib/swarmd"]
   - name: metadata
-    image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d
+    image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5
 services:
   - name: getty
     image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa