diff --git a/blueprints/docker-for-mac/base.yml b/blueprints/docker-for-mac/base.yml index f618a965d..3594e9499 100644 --- a/blueprints/docker-for-mac/base.yml +++ b/blueprints/docker-for-mac/base.yml @@ -11,7 +11,7 @@ init: onboot: # support metadata for optional config in /var/config - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 - name: sysctl image: linuxkit/sysctl:a9ad57ed738a31ea9380cd73236866c312b35489 - name: sysfs diff --git a/blueprints/docker-for-mac/metadata.json b/blueprints/docker-for-mac/metadata.json index c7d11d2bb..c82963ee3 100644 --- a/blueprints/docker-for-mac/metadata.json +++ b/blueprints/docker-for-mac/metadata.json @@ -1,8 +1,10 @@ { "docker": { - "daemon.json": { - "perm": "0644", - "content": "{ \"debug\": true }" + "entries": { + "daemon.json": { + "perm": "0644", + "content": "{ \"debug\": true }" + } } } -} +} \ No newline at end of file diff --git a/docs/metadata.md b/docs/metadata.md index d612c7655..880ebbf20 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -22,19 +22,25 @@ directories to be created and the directories are populated with files. For example, the following userdata file: ```JSON { - "ssh" : { - "sshd_config" : { - "perm" : "0600", - "content": "PermitRootLogin yes\nPasswordAuthentication no" - } - }, - "foo" : { - "bar" : "foobar", - "baz" : { - "perm": "0600", - "content": "bar" - } + "ssh": { + "entries": { + "sshd_config": { + "perm": "0600", + "content": "PermitRootLogin yes\nPasswordAuthentication no" + } } + }, + "foo": { + "entries": { + "bar": { + "content": "foobar" + }, + "baz": { + "perm": "0600", + "content": "bar" + } + } + } } ``` will generate the following files: @@ -44,16 +50,15 @@ will generate the following files: /var/config/foo/baz ``` -Each file can either be: - -- a simple string (as for `foo/bar` above) in which case the file will - be created with the given contents and read/write (but not execute) - permissions for user and read permissions for group and everyone else (in octal format `0644`). -- a map (as for `ssh/sshd_config` and `foo/baz` above) with the - following mandatory keys: - - `content`: the contents of the file. - - `perm`: the permissions to create the file with. +The JSON file consists of a map from `name` to an entry object. Each entry object has the following fields: +- `content`: if present then the entry is a file. The value is a string containing the desired contents of the file. +- `entries`: if present then the entry is a directory. The value is a map from `name` to entry objects. +- `perm`: the permissions to create the file with. +The `content` and `entries` fields are mutually exclusive, it is an error to include both, +one or the other _must_ be present. +The file or directory's name in each case is the same as the key which referred to that entry. + This hierarchy can then be used by individual containers, who can bind mount the config sub-directory into their namespace where it is needed. diff --git a/examples/aws.yml b/examples/aws.yml index 4bfebd531..135f46662 100644 --- a/examples/aws.yml +++ b/examples/aws.yml @@ -13,7 +13,7 @@ onboot: image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: rngd image: linuxkit/rngd:842e5e8ece7934f0cab9fd0027b595ff3471e5b9 diff --git a/examples/gcp.yml b/examples/gcp.yml index d3015e981..71ba28891 100644 --- a/examples/gcp.yml +++ b/examples/gcp.yml @@ -13,7 +13,7 @@ onboot: image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: getty image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa diff --git a/examples/openstack.yml b/examples/openstack.yml index 52f48d553..b3391bd65 100644 --- a/examples/openstack.yml +++ b/examples/openstack.yml @@ -13,7 +13,7 @@ onboot: image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 command: ["/usr/bin/metadata", "openstack"] services: - name: rngd diff --git a/examples/packet.yml b/examples/packet.yml index 625872d3e..ac4731ed3 100644 --- a/examples/packet.yml +++ b/examples/packet.yml @@ -16,7 +16,7 @@ onboot: image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 command: ["/usr/bin/metadata", "packet"] services: - name: rngd diff --git a/examples/vultr.yml b/examples/vultr.yml index 21ea0a40b..53af6e513 100644 --- a/examples/vultr.yml +++ b/examples/vultr.yml @@ -13,7 +13,7 @@ onboot: image: linuxkit/dhcpcd:48831507404049660b960e4055f544917d90378e command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: getty image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa diff --git a/pkg/metadata/main.go b/pkg/metadata/main.go index 05572f8ed..46f3aa617 100644 --- a/pkg/metadata/main.go +++ b/pkg/metadata/main.go @@ -111,7 +111,7 @@ func main() { } if userdata != nil { - if err := processUserData(userdata); err != nil { + if err := processUserData(ConfigPath, userdata); err != nil { log.Printf("Could not extract user data: %s", err) } } @@ -139,70 +139,82 @@ func main() { // } // } // Will create foobar/foo with mode 0644 and content "hello" -func processUserData(data []byte) error { +func processUserData(basePath string, data []byte) error { // Always write the raw data to a file - err := ioutil.WriteFile(path.Join(ConfigPath, "userdata"), data, 0644) + err := ioutil.WriteFile(path.Join(basePath, "userdata"), data, 0644) if err != nil { log.Printf("Could not write userdata: %s", err) return err } - var fd interface{} - if err := json.Unmarshal(data, &fd); err != nil { + var root ConfigFile + if err := json.Unmarshal(data, &root); err != nil { // Userdata is no JSON, presumably... log.Printf("Could not unmarshall userdata: %s", err) // This is not an error return nil } - cm, ok := fd.(map[string]interface{}) - if !ok { - log.Printf("Could convert JSON to desired format: %s", fd) - return nil + + for dir, entry := range root { + writeConfigFiles(path.Join(basePath, dir), entry) } - for d, val := range cm { - dir := path.Join(ConfigPath, d) - if err := os.MkdirAll(dir, 0755); err != nil { - log.Printf("Failed to create %s: %s", dir, err) - continue - } - files, ok := val.(map[string]interface{}) - if !ok { - log.Printf("Could convert JSON for files: %s", val) - continue - } - for f, i := range files { - p := uint64(0644) - var c string - - switch fi := i.(type) { - case map[string]interface{}: - if _, ok := fi["perm"]; !ok { - log.Printf("No permission provided %s", f) - continue - } - if _, ok := fi["content"]; !ok { - log.Printf("No content provided %s", f) - continue - } - c = fi["content"].(string) - if p, err = strconv.ParseUint(fi["perm"].(string), 8, 32); err != nil { - log.Printf("Failed to parse permission %s: %s", fi, err) - continue - } - case string: - c = fi - default: - log.Printf("Couldn't convert JSON for items: %s", i) - continue - } - - if err := ioutil.WriteFile(path.Join(dir, f), []byte(c), os.FileMode(p)); err != nil { - log.Printf("Failed to write %s/%s: %s", dir, f, err) - continue - - } - } - } - return nil } + +func writeConfigFiles(target string, current Entry) { + if isFile(current) { + filemode, err := parseFileMode(current.Perm, 0644) + if err != nil { + log.Printf("Failed to parse permission %s: %s", current, err) + return + } + if err := ioutil.WriteFile(target, []byte(*current.Content), filemode); err != nil { + log.Printf("Failed to write %s: %s", target, err) + return + } + } else if isDirectory(current) { + filemode, err := parseFileMode(current.Perm, 0755) + if err != nil { + log.Printf("Failed to parse permission %s: %s", current, err) + return + } + if err := os.MkdirAll(target, filemode); err != nil { + log.Printf("Failed to create %s: %s", target, err) + return + } + for dir, entry := range current.Entries { + writeConfigFiles(path.Join(target, dir), entry) + } + } else { + log.Printf("%s is invalid", target) + } +} + +func isFile(json Entry) bool { + return json.Content != nil && json.Entries == nil +} + +func isDirectory(json Entry) bool { + return json.Content == nil && json.Entries != nil +} + +func parseFileMode(input string, defaultMode os.FileMode) (os.FileMode, error) { + if input != "" { + perm, err := strconv.ParseUint(input, 8, 32) + if err != nil { + return 0, err + } + return os.FileMode(perm), nil + } + return defaultMode, nil +} + +// ConfigFile represents the configuration file +type ConfigFile map[string]Entry + +// Entry represents either a directory or a file +type Entry struct { + Perm string `json:"perm,omitempty"` + Content *string `json:"content,omitempty"` + Entries map[string]Entry `json:"entries,omitempty"` +} diff --git a/pkg/metadata/main_test.go b/pkg/metadata/main_test.go new file mode 100644 index 000000000..15915eb7e --- /dev/null +++ b/pkg/metadata/main_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "os" + "path" + "testing" +) + +func TestSampleConfig(t *testing.T) { + basePath, err := ioutil.TempDir("", "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "ssh": { + "entries": { + "sshd_config": { + "perm": "0600", + "content": "PermitRootLogin yes\nPasswordAuthentication no" + } + } + }, + "foo": { + "entries": { + "bar": { + "content": "foobar" + }, + "baz": { + "perm": "0600", + "content": "bar" + } + } + } + }`) + + sshd := path.Join(basePath, "ssh", "sshd_config") + assertContent(t, sshd, "PermitRootLogin yes\nPasswordAuthentication no") + assertPermission(t, sshd, 0600) + + bar := path.Join(basePath, "foo", "bar") + assertContent(t, bar, "foobar") + assertPermission(t, bar, 0644) + + assertContent(t, path.Join(basePath, "foo", "baz"), "bar") +} + +func TestSerialization(t *testing.T) { + bin, err := json.Marshal(ConfigFile{ + "ssh": Entry{ + Entries: map[string]Entry{ + "sshd_config": { + Content: str("PermitRootLogin yes\nPasswordAuthentication no"), + Perm: "0600", + }, + }, + }, + "foo": Entry{ + Entries: map[string]Entry{ + "bar": { + Content: str("foobar"), + }, + "baz": { + Content: str("bar"), + Perm: "0600", + }, + }, + }, + }) + if err != nil { + t.Fatalf("Cannot convert to json: %v", err) + } + + expected := `{"foo":{"entries":{"bar":{"content":"foobar"},"baz":{"perm":"0600","content":"bar"}}},"ssh":{"entries":{"sshd_config":{"perm":"0600","content":"PermitRootLogin yes\nPasswordAuthentication no"}}}}` + if expected != string(bin) { + t.Fatalf("Expected %v but has %v", expected, string(bin)) + } +} + +func TestWriteSingleFile(t *testing.T) { + basePath, err := ioutil.TempDir(os.TempDir(), "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "hostname": { + "content": "foobar" + } + }`) + + assertContent(t, path.Join(basePath, "hostname"), "foobar") +} + +func TestWriteEmptyFile(t *testing.T) { + basePath, err := ioutil.TempDir(os.TempDir(), "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "empty": { + "content": "" + } + }`) + + assertContent(t, path.Join(basePath, "empty"), "") +} + +func TestWriteEmptyDirectory(t *testing.T) { + basePath, err := ioutil.TempDir(os.TempDir(), "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "empty": { + "entries": {} + } + }`) + + if _, err := os.Stat(path.Join(basePath, "empty")); err != nil { + t.Fatalf("empty folder doesn't exist: %v", err) + } +} + +func TestSetPermission(t *testing.T) { + basePath, err := ioutil.TempDir(os.TempDir(), "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "restricted": { + "perm": "0600", + "entries": { + "password": { + "perm": "0600", + "content": "secret" + } + } + } + }`) + + assertPermission(t, path.Join(basePath, "restricted"), 0600|os.ModeDir) + assertPermission(t, path.Join(basePath, "restricted", "password"), 0600) +} + +func TestDeepTree(t *testing.T) { + basePath, err := ioutil.TempDir("", "metadata") + if err != nil { + t.Fatalf("can't make a temp rootdir %v", err) + } + defer os.RemoveAll(basePath) + + process(t, basePath, `{ + "level1": { + "entries": { + "level2": { + "entries": { + "file2": { + "content": "depth2" + }, + "level3": { + "entries": { + "file3": { + "content": "depth3" + } + } + } + } + } + } + } + }`) + + assertContent(t, path.Join(basePath, "level1", "level2", "level3", "file3"), "depth3") + assertContent(t, path.Join(basePath, "level1", "level2", "file2"), "depth2") +} + +func str(input string) *string { + return &input +} + +func process(t *testing.T, basePath string, json string) { + if err := processUserData(basePath, []byte(json)); err != nil { + t.Fatalf("fail to process json %v", err) + } +} + +func assertPermission(t *testing.T, path string, expected os.FileMode) { + fileinfo, err := os.Stat(path) + if err != nil { + t.Fatalf("%v doesn't exist: %v", path, err) + } + if fileinfo.Mode() != expected { + t.Fatalf("%v: expected %v but has %v", path, expected, fileinfo.Mode()) + } +} + +func assertContent(t *testing.T, path, expected string) { + file, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("can't read %v: %v", path, err) + } + if !bytes.Equal(file, []byte(expected)) { + t.Fatalf("%v: expected %v but has %v", path, string(expected), string(file)) + } +} diff --git a/projects/etcd/etcd.yml b/projects/etcd/etcd.yml index a5d1abefe..c37a9712c 100644 --- a/projects/etcd/etcd.yml +++ b/projects/etcd/etcd.yml @@ -18,7 +18,7 @@ onboot: image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2 command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3 + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: rngd image: linuxkit/rngd:45ed7759dd927f4cce3863073ea2e0da1d52a427 diff --git a/projects/etcd/prom-us-central1-f.yml b/projects/etcd/prom-us-central1-f.yml index 002067dd9..bc310bf7e 100644 --- a/projects/etcd/prom-us-central1-f.yml +++ b/projects/etcd/prom-us-central1-f.yml @@ -13,7 +13,7 @@ onboot: image: linuxkit/dhcpcd:aa685261ceb2557990dcfe9dd8824c6b9ec416e2 command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - name: metadata - image: linuxkit/metadata:52a3d36ed158357125f3a998f9d03784eb0636d3 + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: rngd image: mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9 diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 07205e6a6..3a6872a2d 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -20,7 +20,7 @@ onboot: image: linuxkit/mount:41685ecc8039643948e5dff46e17584753469a7a command: ["/usr/bin/mountie", "/var/lib/swarmd"] - name: metadata - image: linuxkit/metadata:9506d124d0a3ff645c9781c47f207423abf6154d + image: linuxkit/metadata:026aca5c08c22589a7e319f79449bef2c65f04c5 services: - name: getty image: linuxkit/getty:6af22c32c98536a79230eef000e9abd06b037faa