pkg: Add a generic metadata package

This package handles meta and user data for different cloud
and other platforms. It should be easy to extend to new
platforms.

Currently, it handles GCP metadata and a simple CDROM userdata
provider.

Signed-off-by: Rolf Neugebauer <rolf.neugebauer@docker.com>

pkg/metadata/.gitignore

@@ -0,0 +1,4 @@
+dev
+proc
+sys
+usr

pkg/metadata/Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+COPY . ./
+CMD ["/usr/bin/metadata"]

pkg/metadata/Makefile

@@ -0,0 +1,44 @@
+GO_COMPILE=mobylinux/go-compile:90607983001c2789911afabf420394d51f78ced8@sha256:188beb574d4702a92fa3396a57cabaade28003c82f9413c3121a370ff8becea4
+
+SHA_IMAGE=alpine:3.5@sha256:dfbd4a3a8ebca874ebd2474f044a0b33600d4523d03b0df76e5c5986cb02d7e8
+
+METADATA_BINARY=usr/bin/metadata
+
+IMAGE=metadata
+
+.PHONY: tag push clean container
+default: push
+
+$(METADATA_BINARY): $(wildcard *.go) Makefile
+	mkdir -p $(dir $@)
+	tar cf - $^ | docker run --rm --net=none --log-driver=none -i $(GO_COMPILE) -o $@ | tar xf -
+
+DIRS=dev proc sys
+$(DIRS):
+	mkdir -p $@
+
+DEPS=$(DIRS) $(METADATA_BINARY)
+
+container: Dockerfile $(DEPS)
+	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
+
+hash: Dockerfile $(DEPS)
+	find $^ -type f | xargs cat | docker run --rm -i $(SHA_IMAGE) sha1sum - | sed 's/ .*//' > hash
+
+push: hash container
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
+		 docker push mobylinux/$(IMAGE):$(shell cat hash))
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+tag: hash container
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+clean:
+	rm -rf hash $(DIRS) usr
+
+.DELETE_ON_ERROR:

pkg/metadata/main.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"strconv"
+	"syscall"
+)
+
+const (
+	// ConfigPath is where the data is extracted to
+	ConfigPath = "/var/config"
+
+	// MountPoint is where the CDROM is mounted
+	MountPoint = "/cdrom"
+
+	// Hostname is the filename in configPath where the hostname is stored
+	Hostname = "hostname"
+
+	// SSH is the path where sshd configuration from the provider is stored
+	SSH = "ssh"
+
+	// TODO(rneugeba): Need to check this is the same everywhere
+	cdromDev = "/dev/sr0"
+)
+
+// Provider is a generic interface for metadata/userdata providers.
+type Provider interface {
+	// Probe returns true if the provider was detected.
+	Probe() bool
+
+	// Extract user data. This may write some data, specific to a
+	// provider, to ConfigPath and should return the generic userdata.
+	Extract() ([]byte, error)
+}
+
+// netProviders is a list of Providers offering metadata/userdata over the network
+var netProviders []Provider
+
+// cdromProviders is a list of Providers offering metadata/userdata data via CDROM
+var cdromProviders []Provider
+
+func init() {
+	netProviders = []Provider{NewGCP()}
+	cdromProviders = []Provider{NewCDROM()}
+}
+
+func main() {
+	if err := os.MkdirAll(ConfigPath, 0755); err != nil {
+		log.Fatalf("Could not create %s: %s", ConfigPath, err)
+	}
+
+	var userdata []byte
+	var err error
+	found := false
+	for _, p := range netProviders {
+		if p.Probe() {
+			userdata, err = p.Extract()
+			found = true
+			break
+		}
+	}
+	if !found {
+		log.Printf("Trying CDROM")
+		if err := os.Mkdir(MountPoint, 0755); err != nil {
+			log.Printf("CDROM: Failed to create %s: %s", MountPoint, err)
+			goto ErrorOut
+		}
+		if err := mountCDROM(cdromDev, MountPoint); err != nil {
+			log.Printf("Failed to mount cdrom: %s", err)
+			goto ErrorOut
+		}
+		defer syscall.Unmount(MountPoint, 0)
+		// Don't worry about removing MountPoint. We are in a container
+
+		for _, p := range cdromProviders {
+			if p.Probe() {
+				userdata, err = p.Extract()
+				found = true
+				break
+			}
+		}
+	}
+
+ErrorOut:
+	if !found {
+		log.Printf("No metadata/userdata found. Bye")
+		return
+	}
+
+	if err != nil {
+		log.Printf("Error during metadata probe: %s", err)
+	}
+
+	if userdata != nil {
+		if err := processUserData(userdata); err != nil {
+			log.Printf("Could not extract user data: %s", err)
+		}
+	}
+
+	// Handle setting the hostname as a special case. We want to
+	// do this early and don't really want another container for it.
+	hostname, err := ioutil.ReadFile(path.Join(ConfigPath, Hostname))
+	if err == nil {
+		err := syscall.Sethostname(hostname)
+		if err != nil {
+			log.Printf("Failed to set hostname: %s", err)
+		} else {
+			log.Printf("Set hostname to: %s", string(hostname))
+		}
+	}
+}
+
+// If the userdata is a json file, create a directory/file hierarchy.
+// Example:
+// {
+//    "foobar" : {
+//        "foo" : {
+//            "perm": "0644",
+//            "content": "hello"
+//        }
+// }
+// Will create foobar/foo with mode 0644 and content "hello"
+func processUserData(data []byte) error {
+	// Always write the raw data to a file
+	err := ioutil.WriteFile(path.Join(ConfigPath, "userdata"), data, 0644)
+	if err != nil {
+		log.Printf("Could not write userdata: %s", err)
+		return err
+	}
+
+	var fd interface{}
+	if err := json.Unmarshal(data, &fd); err != nil {
+		// Userdata is no JSON, presumably...
+		log.Printf("Could not unmarshall userdata: %s", err)
+		// This is not an error
+		return nil
+	}
+	cm := fd.(map[string]interface{})
+	for d, val := range cm {
+		dir := path.Join(ConfigPath, d)
+		if err := os.Mkdir(dir, 0755); err != nil {
+			log.Printf("Failed to create %s: %s", dir, err)
+			continue
+		}
+		files := val.(map[string]interface{})
+		for f, i := range files {
+			fi := i.(map[string]interface{})
+			if _, ok := fi["perm"]; !ok {
+				log.Printf("No permission provided %s:%s", f, fi)
+				continue
+			}
+			if _, ok := fi["content"]; !ok {
+				log.Printf("No content provided %s:%s", f, fi)
+				continue
+			}
+			c := fi["content"].(string)
+			p, err := strconv.ParseUint(fi["perm"].(string), 8, 32)
+			if err != nil {
+				log.Printf("Failed to parse permission %s: %s", fi, err)
+				continue
+			}
+			if err := ioutil.WriteFile(path.Join(dir, f), []byte(c), os.FileMode(p)); err != nil {
+				log.Printf("Failed to write %s/%s: %s", dir, f, err)
+				continue
+
+			}
+		}
+	}
+
+	return nil
+}
+
+// mountCDROM mounts a CDROM/DVD device under mountPoint
+func mountCDROM(device, mountPoint string) error {
+	// We may need to poll a little for device ready
+	return syscall.Mount(device, mountPoint, "iso9660", syscall.MS_RDONLY, "")
+}

pkg/metadata/provider_cdrom.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+)
+
+const (
+	configFile = "config"
+)
+
+// ProviderCDROM is the type implementing the Provider interface for CDROMs
+// It looks for a file called 'configFile' in the root
+type ProviderCDROM struct {
+}
+
+// NewCDROM returns a new ProviderCDROM
+func NewCDROM() *ProviderCDROM {
+	return &ProviderCDROM{}
+}
+
+// Probe checks if the CD has the right file
+func (p *ProviderCDROM) Probe() bool {
+	_, err := os.Stat(path.Join(MountPoint, configFile))
+	if err != nil {
+		log.Printf("CDROM: Probe -> %s", err)
+	}
+	return (!os.IsNotExist(err))
+}
+
+// Extract gets both the CDROM specific and generic userdata
+func (p *ProviderCDROM) Extract() ([]byte, error) {
+	data, err := ioutil.ReadFile(path.Join(MountPoint, configFile))
+	if err != nil {
+		return nil, fmt.Errorf("CDROM: Error reading file: %s", err)
+	}
+	return data, nil
+}

pkg/metadata/provider_gcp.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+)
+
+const (
+	project  = "http://metadata.google.internal/computeMetadata/v1/project/"
+	instance = "http://metadata.google.internal/computeMetadata/v1/instance/"
+)
+
+// ProviderGCP is the type implementing the Provider interface for GCP
+type ProviderGCP struct {
+}
+
+// NewGCP returns a new ProviderGCP
+func NewGCP() *ProviderGCP {
+	return &ProviderGCP{}
+}
+
+// Probe checks if we are running on GCP
+func (p *ProviderGCP) Probe() bool {
+	// Getting the hostname should always work...
+	_, err := gcpGet(instance + "hostname")
+	log.Printf("GCP: Probe -> %s", err)
+	return (err == nil)
+}
+
+// Extract gets both the GCP specific and generic userdata
+func (p *ProviderGCP) Extract() ([]byte, error) {
+	// Get host name. This must not fail
+	hostname, err := gcpGet(instance + "hostname")
+	if err != nil {
+		return nil, err
+	}
+	err = ioutil.WriteFile(path.Join(ConfigPath, Hostname), hostname, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("GCP: Failed to write hostname: %s", err)
+	}
+
+	// SSH keys:
+	// TODO also retrieve the instance keys and respect block
+	//      project keys see:
+	//      https://cloud.google.com/compute/docs/instances/ssh-keys
+	// The keys have usernames attached, but as a simplification
+	// we are going to add them all to one root file
+	// TODO split them into individual user files and make the ssh
+	//      container construct those users
+	sshKeys, err := gcpGet(project + "attributes/sshKeys")
+	if err == nil {
+		rootKeys := ""
+		for _, line := range strings.Split(string(sshKeys), "\n") {
+			parts := strings.SplitN(line, ":", 2)
+			// ignoring username for now
+			if len(parts) == 2 {
+				rootKeys = rootKeys + parts[1] + "\n"
+			}
+		}
+		err = ioutil.WriteFile(path.Join(ConfigPath, SSH, "authorized_keys"), []byte(rootKeys), 0600)
+		if err != nil {
+			log.Printf("GCP: Failed to write ssh keys: %s", err)
+		}
+	}
+
+	// Generic userdata
+	userData, err := gcpGet(instance + "attributes/userdata")
+	if err != nil {
+		log.Printf("GCP: Failed to get user-data: %s", err)
+		// This is not an error
+		return nil, nil
+	}
+	return userData, nil
+}
+
+// gcpGet requests and extracts the requested URL
+func gcpGet(url string) ([]byte, error) {
+	var client = &http.Client{
+		Timeout: time.Second * 2,
+	}
+
+	req, err := http.NewRequest("", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("GCP: http.NewRequest failed: %s", err)
+	}
+	req.Header.Set("Metadata-Flavor", "Google")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("GCP: Could not contact metadata service: %s", err)
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("GCP: Status not ok: %d", resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("GCP: Failed to read http response: %s", err)
+	}
+	return body, nil
+}