Add aliyun datasource (#2169)

cmd/cloudinitsave/cloudinitsave.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/configdrive"
 	"github.com/rancher/os/config/cloudinit/datasource/file"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/aliyun"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/digitalocean"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/ec2"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/gce"
@@ -264,6 +265,8 @@ func getDatasources(datasources []string) []datasource.Datasource {
 			if v != nil {
 				dss = append(dss, v)
 			}
+		case "aliyun":
+			dss = append(dss, aliyun.NewDatasource(root))
 		}
 	}
 

config/cloudinit/datasource/metadata/aliyun/metadata.go

@@ -0,0 +1,85 @@
+package aliyun
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/rancher/os/netconf"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+)
+
+const (
+	DefaultAddress = "http://100.100.100.200/"
+	apiVersion     = "2016-01-01/"
+	userdataPath   = apiVersion + "user-data/"
+	metadataPath   = apiVersion + "meta-data/"
+)
+
+type MetadataService struct {
+	metadata.Service
+}
+
+func NewDatasource(root string) *MetadataService {
+	if root == "" {
+		root = DefaultAddress
+	}
+	return &MetadataService{metadata.NewDatasource(root, apiVersion, userdataPath, metadataPath, nil)}
+}
+
+func (ms MetadataService) AvailabilityChanges() bool {
+	// TODO: if it can't find the network, maybe we can start it?
+	return false
+}
+
+func (ms MetadataService) FetchMetadata() (metadata datasource.Metadata, err error) {
+	// see https://www.alibabacloud.com/help/faq-detail/49122.htm
+	metadata.NetworkConfig = netconf.NetworkConfig{}
+
+	enablePublicKey := false
+
+	rootContents, err := ms.FetchAttributes("")
+	if err != nil {
+		return metadata, err
+	}
+	for _, c := range rootContents {
+		if c == "public-keys/" {
+			enablePublicKey = true
+			break
+		}
+	}
+	if !enablePublicKey {
+		return metadata, fmt.Errorf("The public-keys should be enable in %s", ms.Type())
+	}
+
+	keynames, err := ms.FetchAttributes("public-keys/")
+	if err != nil {
+		return metadata, err
+	}
+
+	metadata.SSHPublicKeys = map[string]string{}
+	for _, k := range keynames {
+		k = strings.TrimRight(k, "/")
+		sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", k))
+		if err != nil {
+			return metadata, err
+		}
+		metadata.SSHPublicKeys[k] = sshkey
+		log.Printf("Found SSH key for %q\n", k)
+	}
+
+	if hostname, err := ms.FetchAttribute("hostname"); err == nil {
+		metadata.Hostname = hostname
+		log.Printf("Found hostname  %s\n", hostname)
+	} else {
+		return metadata, err
+	}
+
+	return metadata, nil
+}
+
+func (ms MetadataService) Type() string {
+	return "aliyun-metadata-service"
+}

config/cloudinit/datasource/metadata/aliyun/metadata_test.go

@@ -0,0 +1,78 @@
+package aliyun
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
+	"github.com/rancher/os/config/cloudinit/pkg"
+)
+
+func TestType(t *testing.T) {
+	want := "aliyun-metadata-service"
+	if kind := (MetadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       datasource.Metadata
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root:         "/",
+			metadataPath: "2016-01-01/meta-data/",
+			resources: map[string]string{
+				"/2016-01-01/meta-data/": "hostname\n",
+			},
+			expectErr: fmt.Errorf("The public-keys should be enable in aliyun-metadata-service"),
+		},
+		{
+			root:         "/",
+			metadataPath: "2016-01-01/meta-data/",
+			resources: map[string]string{
+				"/2016-01-01/meta-data/":                           "hostname\npublic-keys/\n",
+				"/2016-01-01/meta-data/hostname":                   "host",
+				"/2016-01-01/meta-data/public-keys/":               "xx/",
+				"/2016-01-01/meta-data/public-keys/xx/":            "openssh-key",
+				"/2016-01-01/meta-data/public-keys/xx/openssh-key": "key",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				SSHPublicKeys: map[string]string{"xx": "key"},
+			},
+		},
+		{
+			clientErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+		},
+	} {
+		service := &MetadataService{metadata.Service{
+			Root:         tt.root,
+			Client:       &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			MetadataPath: tt.metadataPath,
+		}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): \nwant %q, \ngot %q\n", tt.resources, tt.expectErr, err)
+		}
+		if !reflect.DeepEqual(tt.expect, metadata) {
+			t.Fatalf("bad fetch (%q): \nwant %#v, \ngot %#v\n", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

config/cloudinit/datasource/metadata/ec2/metadata.go

@@ -15,8 +15,6 @@
 package ec2
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
 	"log"
 	"net"
@@ -57,7 +55,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 	metadata := datasource.Metadata{}
 	metadata.NetworkConfig = netconf.NetworkConfig{}
 
-	if keynames, err := ms.fetchAttributes("public-keys"); err == nil {
+	if keynames, err := ms.FetchAttributes("public-keys"); err == nil {
 		keyIDs := make(map[string]string)
 		for _, keyname := range keynames {
 			tokens := strings.SplitN(keyname, "=", 2)
@@ -69,7 +67,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 
 		metadata.SSHPublicKeys = map[string]string{}
 		for name, id := range keyIDs {
-			sshkey, err := ms.fetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id))
+			sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id))
 			if err != nil {
 				return metadata, err
 			}
@@ -80,44 +78,44 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 		return metadata, err
 	}
 
-	if hostname, err := ms.fetchAttribute("hostname"); err == nil {
+	if hostname, err := ms.FetchAttribute("hostname"); err == nil {
 		metadata.Hostname = strings.Split(hostname, " ")[0]
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
 
 	// TODO: these are only on the first interface - it looks like you can have as many as you need...
-	if localAddr, err := ms.fetchAttribute("local-ipv4"); err == nil {
+	if localAddr, err := ms.FetchAttribute("local-ipv4"); err == nil {
 		metadata.PrivateIPv4 = net.ParseIP(localAddr)
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
-	if publicAddr, err := ms.fetchAttribute("public-ipv4"); err == nil {
+	if publicAddr, err := ms.FetchAttribute("public-ipv4"); err == nil {
 		metadata.PublicIPv4 = net.ParseIP(publicAddr)
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
 
 	metadata.NetworkConfig.Interfaces = make(map[string]netconf.InterfaceConfig)
-	if macs, err := ms.fetchAttributes("network/interfaces/macs"); err != nil {
+	if macs, err := ms.FetchAttributes("network/interfaces/macs"); err != nil {
 		for _, mac := range macs {
-			if deviceNumber, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil {
+			if deviceNumber, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil {
 				network := netconf.InterfaceConfig{
 					DHCP: true,
 				}
 				/* Looks like we must use DHCP for aws
 				// private ipv4
-				if subnetCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil {
+				if subnetCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil {
 					cidr := strings.Split(subnetCidrBlock, "/")
-					if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil {
+					if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil {
 						for _, addr := range localAddr {
 							network.Addresses = append(network.Addresses, addr+"/"+cidr[1])
 						}
 					}
 				}
 				// ipv6
-				if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil {
-					if subnetCidrBlock, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil {
+				if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil {
+					if subnetCidrBlock, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil {
 						for i, addr := range localAddr {
 							cidr := strings.Split(subnetCidrBlock[i], "/")
 							network.Addresses = append(network.Addresses, addr+"/"+cidr[1])
@@ -126,8 +124,8 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 				}
 				*/
 				// disabled - it looks to me like you don't actually put the public IP on the eth device
-				/*				if publicAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil {
-									if vpcCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil {
+				/*				if publicAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil {
+									if vpcCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil {
 										cidr := strings.Split(vpcCidrBlock, "/")
 										network.Addresses = append(network.Addresses, publicAddr+"/"+cidr[1])
 									}
@@ -145,25 +143,3 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 func (ms MetadataService) Type() string {
 	return "ec2-metadata-service"
 }
-
-func (ms MetadataService) fetchAttributes(key string) ([]string, error) {
-	url := ms.MetadataURL() + key
-	resp, err := ms.FetchData(url)
-	if err != nil {
-		return nil, err
-	}
-	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
-	data := make([]string, 0)
-	for scanner.Scan() {
-		data = append(data, scanner.Text())
-	}
-	return data, scanner.Err()
-}
-
-func (ms MetadataService) fetchAttribute(key string) (string, error) {
-	attrs, err := ms.fetchAttributes(key)
-	if err == nil && len(attrs) > 0 {
-		return attrs[0], nil
-	}
-	return "", err
-}

config/cloudinit/datasource/metadata/ec2/metadata_test.go

@@ -34,114 +34,6 @@ func TestType(t *testing.T) {
 	}
 }
 
-func TestFetchAttributes(t *testing.T) {
-	for _, s := range []struct {
-		resources map[string]string
-		err       error
-		tests     []struct {
-			path string
-			val  []string
-		}
-	}{
-		{
-			resources: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"/", []string{"a", "b", "c/"}},
-				{"/b", []string{"2"}},
-				{"/c/d", []string{"3"}},
-				{"/c/e/", []string{"f"}},
-			},
-		},
-		{
-			err: fmt.Errorf("test error"),
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"", nil},
-			},
-		},
-	} {
-		service := MetadataService{metadata.Service{
-			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
-		}}
-		for _, tt := range s.tests {
-			attrs, err := service.fetchAttributes(tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
-			}
-			if !reflect.DeepEqual(attrs, tt.val) {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
-			}
-		}
-	}
-}
-
-func TestFetchAttribute(t *testing.T) {
-	for _, s := range []struct {
-		resources map[string]string
-		err       error
-		tests     []struct {
-			path string
-			val  string
-		}
-	}{
-		{
-			resources: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"/a", "1"},
-				{"/b", "2"},
-				{"/c/d", "3"},
-				{"/c/e/f", "4"},
-			},
-		},
-		{
-			err: fmt.Errorf("test error"),
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"", ""},
-			},
-		},
-	} {
-		service := MetadataService{metadata.Service{
-			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
-		}}
-		for _, tt := range s.tests {
-			attr, err := service.fetchAttribute(tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
-			}
-			if attr != tt.val {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
-			}
-		}
-	}
-}
-
 func TestFetchMetadata(t *testing.T) {
 	for _, tt := range []struct {
 		root         string

config/cloudinit/datasource/metadata/metadata.go

@@ -15,6 +15,8 @@
 package metadata
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"net/http"
 	"strings"
@@ -84,3 +86,25 @@ func (ms Service) MetadataURL() string {
 func (ms Service) UserdataURL() string {
 	return (ms.Root + ms.UserdataPath)
 }
+
+func (ms Service) FetchAttributes(key string) ([]string, error) {
+	url := ms.MetadataURL() + key
+	resp, err := ms.FetchData(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, scanner.Text())
+	}
+	return data, scanner.Err()
+}
+
+func (ms Service) FetchAttribute(key string) (string, error) {
+	attrs, err := ms.FetchAttributes(key)
+	if err == nil && len(attrs) > 0 {
+		return attrs[0], nil
+	}
+	return "", err
+}

config/cloudinit/datasource/metadata/metadata_test.go

@@ -17,6 +17,7 @@ package metadata
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
@@ -177,6 +178,114 @@ func TestNewDatasource(t *testing.T) {
 	}
 }
 
+func TestFetchAttributes(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  []string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"/", []string{"a", "b", "c/"}},
+				{"/b", []string{"2"}},
+				{"/c/d", []string{"3"}},
+				{"/c/e/", []string{"f"}},
+			},
+		},
+		{
+			err: fmt.Errorf("test error"),
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"", nil},
+			},
+		},
+	} {
+		service := &Service{
+			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
+		}
+		for _, tt := range s.tests {
+			attrs, err := service.FetchAttributes(tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if !reflect.DeepEqual(attrs, tt.val) {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
+			}
+		}
+	}
+}
+
+func TestFetchAttribute(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"/a", "1"},
+				{"/b", "2"},
+				{"/c/d", "3"},
+				{"/c/e/f", "4"},
+			},
+		},
+		{
+			err: fmt.Errorf("test error"),
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"", ""},
+			},
+		},
+	} {
+		service := &Service{
+			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
+		}
+		for _, tt := range s.tests {
+			attr, err := service.FetchAttribute(tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if attr != tt.val {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
+			}
+		}
+	}
+}
+
 func Error(err error) string {
 	if err != nil {
 		return err.Error()