metadata: handle json with more than 2 levels

Signed-off-by: Guillaume Rose <guillaume.rose@docker.com>

blueprints/docker-for-mac/metadata.json

@@ -1,8 +1,10 @@
 {
   "docker": {
-    "daemon.json": {
-      "perm": "0644",
-      "content": "{ \"debug\": true }"
+    "entries": {
+      "daemon.json": {
+        "perm": "0644",
+        "content": "{ \"debug\": true }"
+      }
     }
   }
-}
+}

docs/metadata.md

@@ -22,19 +22,25 @@ directories to be created and the directories are populated with files.
 For example, the following userdata file:
 ```JSON
 {
-    "ssh" : {
-        "sshd_config" : {
-            "perm" : "0600",
-            "content": "PermitRootLogin yes\nPasswordAuthentication no"
-        }
-    },
-    "foo" : {
-        "bar" : "foobar",
-        "baz" : {
-            "perm": "0600",
-            "content": "bar"
-        }
+  "ssh": {
+    "entries": {
+      "sshd_config": {
+        "perm": "0600",
+        "content": "PermitRootLogin yes\nPasswordAuthentication no"
+      }
     }
+  },
+  "foo": {
+    "entries": {
+      "bar": {
+        "content": "foobar"
+      },
+      "baz": {
+        "perm": "0600",
+        "content": "bar"
+      }
+    }
+  }
 }
 ```
 will generate the following files:
@@ -44,16 +50,15 @@ will generate the following files:
 /var/config/foo/baz
 ```
 
-Each file can either be:
-
-- a simple string (as for `foo/bar` above) in which case the file will
-  be created with the given contents and read/write (but not execute)
-  permissions for user and read permissions for group and everyone else (in octal format `0644`).
-- a map (as for `ssh/sshd_config` and `foo/baz` above) with the
-  following mandatory keys:
-  - `content`: the contents of the file.
-  - `perm`: the permissions to create the file with.
+The JSON file consists of a map from `name` to an entry object. Each entry object has the following fields:
+- `content`: if present then the entry is a file. The value is a string containing the desired contents of the file.
+- `entries`: if present then the entry is a directory. The value is a map from `name` to entry objects.
+- `perm`: the permissions to create the file with.
 
+The `content` and `entries` fields are mutually exclusive, it is an error to include both, 
+one or the other _must_ be present. 
+The file or directory's name in each case is the same as the key which referred to that entry.
+ 
 This hierarchy can then be used by individual containers, who can bind
 mount the config sub-directory into their namespace where it is
 needed.

pkg/metadata/main.go

@@ -111,7 +111,7 @@ func main() {
 	}
 
 	if userdata != nil {
-		if err := processUserData(userdata); err != nil {
+		if err := processUserData(ConfigPath, userdata); err != nil {
 			log.Printf("Could not extract user data: %s", err)
 		}
 	}
@@ -139,70 +139,82 @@ func main() {
 //        }
 // }
 // Will create foobar/foo with mode 0644 and content "hello"
-func processUserData(data []byte) error {
+func processUserData(basePath string, data []byte) error {
 	// Always write the raw data to a file
-	err := ioutil.WriteFile(path.Join(ConfigPath, "userdata"), data, 0644)
+	err := ioutil.WriteFile(path.Join(basePath, "userdata"), data, 0644)
 	if err != nil {
 		log.Printf("Could not write userdata: %s", err)
 		return err
 	}
 
-	var fd interface{}
-	if err := json.Unmarshal(data, &fd); err != nil {
+	var root ConfigFile
+	if err := json.Unmarshal(data, &root); err != nil {
 		// Userdata is no JSON, presumably...
 		log.Printf("Could not unmarshall userdata: %s", err)
 		// This is not an error
 		return nil
 	}
-	cm, ok := fd.(map[string]interface{})
-	if !ok {
-		log.Printf("Could convert JSON to desired format: %s", fd)
-		return nil
+
+	for dir, entry := range root {
+		writeConfigFiles(path.Join(basePath, dir), entry)
 	}
-	for d, val := range cm {
-		dir := path.Join(ConfigPath, d)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			log.Printf("Failed to create %s: %s", dir, err)
-			continue
-		}
-		files, ok := val.(map[string]interface{})
-		if !ok {
-			log.Printf("Could convert JSON for files: %s", val)
-			continue
-		}
-		for f, i := range files {
-			p := uint64(0644)
-			var c string
-
-			switch fi := i.(type) {
-			case map[string]interface{}:
-				if _, ok := fi["perm"]; !ok {
-					log.Printf("No permission provided %s", f)
-					continue
-				}
-				if _, ok := fi["content"]; !ok {
-					log.Printf("No content provided %s", f)
-					continue
-				}
-				c = fi["content"].(string)
-				if p, err = strconv.ParseUint(fi["perm"].(string), 8, 32); err != nil {
-					log.Printf("Failed to parse permission %s: %s", fi, err)
-					continue
-				}
-			case string:
-				c = fi
-			default:
-				log.Printf("Couldn't convert JSON for items: %s", i)
-				continue
-			}
-
-			if err := ioutil.WriteFile(path.Join(dir, f), []byte(c), os.FileMode(p)); err != nil {
-				log.Printf("Failed to write %s/%s: %s", dir, f, err)
-				continue
-
-			}
-		}
-	}
-
 	return nil
 }
+
+func writeConfigFiles(target string, current Entry) {
+	if isFile(current) {
+		filemode, err := parseFileMode(current.Perm, 0644)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := ioutil.WriteFile(target, []byte(*current.Content), filemode); err != nil {
+			log.Printf("Failed to write %s: %s", target, err)
+			return
+		}
+	} else if isDirectory(current) {
+		filemode, err := parseFileMode(current.Perm, 0755)
+		if err != nil {
+			log.Printf("Failed to parse permission %s: %s", current, err)
+			return
+		}
+		if err := os.MkdirAll(target, filemode); err != nil {
+			log.Printf("Failed to create %s: %s", target, err)
+			return
+		}
+		for dir, entry := range current.Entries {
+			writeConfigFiles(path.Join(target, dir), entry)
+		}
+	} else {
+		log.Printf("%s is invalid", target)
+	}
+}
+
+func isFile(json Entry) bool {
+	return json.Content != nil && json.Entries == nil
+}
+
+func isDirectory(json Entry) bool {
+	return json.Content == nil && json.Entries != nil
+}
+
+func parseFileMode(input string, defaultMode os.FileMode) (os.FileMode, error) {
+	if input != "" {
+		perm, err := strconv.ParseUint(input, 8, 32)
+		if err != nil {
+			return 0, err
+		}
+		return os.FileMode(perm), nil
+	}
+	return defaultMode, nil
+}
+
+// ConfigFile represents the configuration file
+type ConfigFile map[string]Entry
+
+// Entry represents either a directory or a file
+type Entry struct {
+	Perm    string           `json:"perm,omitempty"`
+	Content *string          `json:"content,omitempty"`
+	Entries map[string]Entry `json:"entries,omitempty"`
+}

pkg/metadata/main_test.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestSampleConfig(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "ssh": {
+		"entries": {
+		  "sshd_config": {
+			"perm": "0600",
+			"content": "PermitRootLogin yes\nPasswordAuthentication no"
+		  }
+		}
+	  },
+	  "foo": {
+		"entries": {
+		  "bar": {
+			"content": "foobar"
+		  },
+		  "baz": {
+			"perm": "0600",
+			"content": "bar"
+		  }
+		}
+	  }
+	}`)
+
+	sshd := path.Join(basePath, "ssh", "sshd_config")
+	assertContent(t, sshd, "PermitRootLogin yes\nPasswordAuthentication no")
+	assertPermission(t, sshd, 0600)
+
+	bar := path.Join(basePath, "foo", "bar")
+	assertContent(t, bar, "foobar")
+	assertPermission(t, bar, 0644)
+
+	assertContent(t, path.Join(basePath, "foo", "baz"), "bar")
+}
+
+func TestSerialization(t *testing.T) {
+	bin, err := json.Marshal(ConfigFile{
+		"ssh": Entry{
+			Entries: map[string]Entry{
+				"sshd_config": {
+					Content: str("PermitRootLogin yes\nPasswordAuthentication no"),
+					Perm:    "0600",
+				},
+			},
+		},
+		"foo": Entry{
+			Entries: map[string]Entry{
+				"bar": {
+					Content: str("foobar"),
+				},
+				"baz": {
+					Content: str("bar"),
+					Perm:    "0600",
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Cannot convert to json: %v", err)
+	}
+
+	expected := `{"foo":{"entries":{"bar":{"content":"foobar"},"baz":{"perm":"0600","content":"bar"}}},"ssh":{"entries":{"sshd_config":{"perm":"0600","content":"PermitRootLogin yes\nPasswordAuthentication no"}}}}`
+	if expected != string(bin) {
+		t.Fatalf("Expected %v but has %v", expected, string(bin))
+	}
+}
+
+func TestWriteSingleFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "hostname": {
+		"content": "foobar"
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "hostname"), "foobar")
+}
+
+func TestWriteEmptyFile(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"content": ""
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "empty"), "")
+}
+
+func TestWriteEmptyDirectory(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "empty": {
+		"entries": {}
+	  }
+	}`)
+
+	if _, err := os.Stat(path.Join(basePath, "empty")); err != nil {
+		t.Fatalf("empty folder doesn't exist: %v", err)
+	}
+}
+
+func TestSetPermission(t *testing.T) {
+	basePath, err := ioutil.TempDir(os.TempDir(), "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "restricted": {
+		"perm": "0600",
+		"entries": {
+		  "password": {
+			"perm": "0600",
+			"content": "secret"
+		  }
+		}
+	  }
+	}`)
+
+	assertPermission(t, path.Join(basePath, "restricted"), 0600|os.ModeDir)
+	assertPermission(t, path.Join(basePath, "restricted", "password"), 0600)
+}
+
+func TestDeepTree(t *testing.T) {
+	basePath, err := ioutil.TempDir("", "metadata")
+	if err != nil {
+		t.Fatalf("can't make a temp rootdir %v", err)
+	}
+	defer os.RemoveAll(basePath)
+
+	process(t, basePath, `{
+	  "level1": {
+		"entries": {
+		  "level2": {
+			"entries": {
+			  "file2": {
+				"content": "depth2"
+			  },
+			  "level3": {
+				"entries": {
+				  "file3": {
+					"content": "depth3"
+				  }
+				}
+			  }
+			}
+		  }
+		}
+	  }
+	}`)
+
+	assertContent(t, path.Join(basePath, "level1", "level2", "level3", "file3"), "depth3")
+	assertContent(t, path.Join(basePath, "level1", "level2", "file2"), "depth2")
+}
+
+func str(input string) *string {
+	return &input
+}
+
+func process(t *testing.T, basePath string, json string) {
+	if err := processUserData(basePath, []byte(json)); err != nil {
+		t.Fatalf("fail to process json %v", err)
+	}
+}
+
+func assertPermission(t *testing.T, path string, expected os.FileMode) {
+	fileinfo, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("%v doesn't exist: %v", path, err)
+	}
+	if fileinfo.Mode() != expected {
+		t.Fatalf("%v: expected %v but has %v", path, expected, fileinfo.Mode())
+	}
+}
+
+func assertContent(t *testing.T, path, expected string) {
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Fatalf("can't read %v: %v", path, err)
+	}
+	if !bytes.Equal(file, []byte(expected)) {
+		t.Fatalf("%v: expected %v but has %v", path, string(expected), string(file))
+	}
+}