From 60909e435ff321eae73dfba4fc36d7baf4e48fc7 Mon Sep 17 00:00:00 2001 From: niusmallnan Date: Tue, 19 Dec 2017 14:33:44 +0800 Subject: [PATCH] Add aliyun datasource (#2169) --- cmd/cloudinitsave/cloudinitsave.go | 3 + .../datasource/metadata/aliyun/metadata.go | 85 ++++++++++++++ .../metadata/aliyun/metadata_test.go | 78 +++++++++++++ .../datasource/metadata/ec2/metadata.go | 50 +++----- .../datasource/metadata/ec2/metadata_test.go | 108 ----------------- .../cloudinit/datasource/metadata/metadata.go | 24 ++++ .../datasource/metadata/metadata_test.go | 109 ++++++++++++++++++ 7 files changed, 312 insertions(+), 145 deletions(-) create mode 100644 config/cloudinit/datasource/metadata/aliyun/metadata.go create mode 100644 config/cloudinit/datasource/metadata/aliyun/metadata_test.go diff --git a/cmd/cloudinitsave/cloudinitsave.go b/cmd/cloudinitsave/cloudinitsave.go index 83a5861d..b9e96f0d 100755 --- a/cmd/cloudinitsave/cloudinitsave.go +++ b/cmd/cloudinitsave/cloudinitsave.go @@ -33,6 +33,7 @@ import ( "github.com/rancher/os/config/cloudinit/datasource" "github.com/rancher/os/config/cloudinit/datasource/configdrive" "github.com/rancher/os/config/cloudinit/datasource/file" + "github.com/rancher/os/config/cloudinit/datasource/metadata/aliyun" "github.com/rancher/os/config/cloudinit/datasource/metadata/digitalocean" "github.com/rancher/os/config/cloudinit/datasource/metadata/ec2" "github.com/rancher/os/config/cloudinit/datasource/metadata/gce" @@ -264,6 +265,8 @@ func getDatasources(datasources []string) []datasource.Datasource { if v != nil { dss = append(dss, v) } + case "aliyun": + dss = append(dss, aliyun.NewDatasource(root)) } } diff --git a/config/cloudinit/datasource/metadata/aliyun/metadata.go b/config/cloudinit/datasource/metadata/aliyun/metadata.go new file mode 100644 index 00000000..539bfe05 --- /dev/null +++ b/config/cloudinit/datasource/metadata/aliyun/metadata.go @@ -0,0 +1,85 @@ +package aliyun + +import ( + "fmt" + "log" + "strings" + + "github.com/rancher/os/netconf" + + "github.com/rancher/os/config/cloudinit/datasource" + "github.com/rancher/os/config/cloudinit/datasource/metadata" +) + +const ( + DefaultAddress = "http://100.100.100.200/" + apiVersion = "2016-01-01/" + userdataPath = apiVersion + "user-data/" + metadataPath = apiVersion + "meta-data/" +) + +type MetadataService struct { + metadata.Service +} + +func NewDatasource(root string) *MetadataService { + if root == "" { + root = DefaultAddress + } + return &MetadataService{metadata.NewDatasource(root, apiVersion, userdataPath, metadataPath, nil)} +} + +func (ms MetadataService) AvailabilityChanges() bool { + // TODO: if it can't find the network, maybe we can start it? + return false +} + +func (ms MetadataService) FetchMetadata() (metadata datasource.Metadata, err error) { + // see https://www.alibabacloud.com/help/faq-detail/49122.htm + metadata.NetworkConfig = netconf.NetworkConfig{} + + enablePublicKey := false + + rootContents, err := ms.FetchAttributes("") + if err != nil { + return metadata, err + } + for _, c := range rootContents { + if c == "public-keys/" { + enablePublicKey = true + break + } + } + if !enablePublicKey { + return metadata, fmt.Errorf("The public-keys should be enable in %s", ms.Type()) + } + + keynames, err := ms.FetchAttributes("public-keys/") + if err != nil { + return metadata, err + } + + metadata.SSHPublicKeys = map[string]string{} + for _, k := range keynames { + k = strings.TrimRight(k, "/") + sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", k)) + if err != nil { + return metadata, err + } + metadata.SSHPublicKeys[k] = sshkey + log.Printf("Found SSH key for %q\n", k) + } + + if hostname, err := ms.FetchAttribute("hostname"); err == nil { + metadata.Hostname = hostname + log.Printf("Found hostname %s\n", hostname) + } else { + return metadata, err + } + + return metadata, nil +} + +func (ms MetadataService) Type() string { + return "aliyun-metadata-service" +} diff --git a/config/cloudinit/datasource/metadata/aliyun/metadata_test.go b/config/cloudinit/datasource/metadata/aliyun/metadata_test.go new file mode 100644 index 00000000..1d9b2bb0 --- /dev/null +++ b/config/cloudinit/datasource/metadata/aliyun/metadata_test.go @@ -0,0 +1,78 @@ +package aliyun + +import ( + "fmt" + "reflect" + "testing" + + "github.com/rancher/os/config/cloudinit/datasource" + "github.com/rancher/os/config/cloudinit/datasource/metadata" + "github.com/rancher/os/config/cloudinit/datasource/metadata/test" + "github.com/rancher/os/config/cloudinit/pkg" +) + +func TestType(t *testing.T) { + want := "aliyun-metadata-service" + if kind := (MetadataService{}).Type(); kind != want { + t.Fatalf("bad type: want %q, got %q", want, kind) + } +} + +func TestFetchMetadata(t *testing.T) { + for _, tt := range []struct { + root string + metadataPath string + resources map[string]string + expect datasource.Metadata + clientErr error + expectErr error + }{ + { + root: "/", + metadataPath: "2016-01-01/meta-data/", + resources: map[string]string{ + "/2016-01-01/meta-data/": "hostname\n", + }, + expectErr: fmt.Errorf("The public-keys should be enable in aliyun-metadata-service"), + }, + { + root: "/", + metadataPath: "2016-01-01/meta-data/", + resources: map[string]string{ + "/2016-01-01/meta-data/": "hostname\npublic-keys/\n", + "/2016-01-01/meta-data/hostname": "host", + "/2016-01-01/meta-data/public-keys/": "xx/", + "/2016-01-01/meta-data/public-keys/xx/": "openssh-key", + "/2016-01-01/meta-data/public-keys/xx/openssh-key": "key", + }, + expect: datasource.Metadata{ + Hostname: "host", + SSHPublicKeys: map[string]string{"xx": "key"}, + }, + }, + { + clientErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")}, + expectErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")}, + }, + } { + service := &MetadataService{metadata.Service{ + Root: tt.root, + Client: &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr}, + MetadataPath: tt.metadataPath, + }} + metadata, err := service.FetchMetadata() + if Error(err) != Error(tt.expectErr) { + t.Fatalf("bad error (%q): \nwant %q, \ngot %q\n", tt.resources, tt.expectErr, err) + } + if !reflect.DeepEqual(tt.expect, metadata) { + t.Fatalf("bad fetch (%q): \nwant %#v, \ngot %#v\n", tt.resources, tt.expect, metadata) + } + } +} + +func Error(err error) string { + if err != nil { + return err.Error() + } + return "" +} diff --git a/config/cloudinit/datasource/metadata/ec2/metadata.go b/config/cloudinit/datasource/metadata/ec2/metadata.go index 8ef35c92..fbf0de06 100755 --- a/config/cloudinit/datasource/metadata/ec2/metadata.go +++ b/config/cloudinit/datasource/metadata/ec2/metadata.go @@ -15,8 +15,6 @@ package ec2 import ( - "bufio" - "bytes" "fmt" "log" "net" @@ -57,7 +55,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) { metadata := datasource.Metadata{} metadata.NetworkConfig = netconf.NetworkConfig{} - if keynames, err := ms.fetchAttributes("public-keys"); err == nil { + if keynames, err := ms.FetchAttributes("public-keys"); err == nil { keyIDs := make(map[string]string) for _, keyname := range keynames { tokens := strings.SplitN(keyname, "=", 2) @@ -69,7 +67,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) { metadata.SSHPublicKeys = map[string]string{} for name, id := range keyIDs { - sshkey, err := ms.fetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id)) + sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id)) if err != nil { return metadata, err } @@ -80,44 +78,44 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) { return metadata, err } - if hostname, err := ms.fetchAttribute("hostname"); err == nil { + if hostname, err := ms.FetchAttribute("hostname"); err == nil { metadata.Hostname = strings.Split(hostname, " ")[0] } else if _, ok := err.(pkg.ErrNotFound); !ok { return metadata, err } // TODO: these are only on the first interface - it looks like you can have as many as you need... - if localAddr, err := ms.fetchAttribute("local-ipv4"); err == nil { + if localAddr, err := ms.FetchAttribute("local-ipv4"); err == nil { metadata.PrivateIPv4 = net.ParseIP(localAddr) } else if _, ok := err.(pkg.ErrNotFound); !ok { return metadata, err } - if publicAddr, err := ms.fetchAttribute("public-ipv4"); err == nil { + if publicAddr, err := ms.FetchAttribute("public-ipv4"); err == nil { metadata.PublicIPv4 = net.ParseIP(publicAddr) } else if _, ok := err.(pkg.ErrNotFound); !ok { return metadata, err } metadata.NetworkConfig.Interfaces = make(map[string]netconf.InterfaceConfig) - if macs, err := ms.fetchAttributes("network/interfaces/macs"); err != nil { + if macs, err := ms.FetchAttributes("network/interfaces/macs"); err != nil { for _, mac := range macs { - if deviceNumber, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil { + if deviceNumber, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil { network := netconf.InterfaceConfig{ DHCP: true, } /* Looks like we must use DHCP for aws // private ipv4 - if subnetCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil { + if subnetCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil { cidr := strings.Split(subnetCidrBlock, "/") - if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil { + if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil { for _, addr := range localAddr { network.Addresses = append(network.Addresses, addr+"/"+cidr[1]) } } } // ipv6 - if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil { - if subnetCidrBlock, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil { + if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil { + if subnetCidrBlock, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil { for i, addr := range localAddr { cidr := strings.Split(subnetCidrBlock[i], "/") network.Addresses = append(network.Addresses, addr+"/"+cidr[1]) @@ -126,8 +124,8 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) { } */ // disabled - it looks to me like you don't actually put the public IP on the eth device - /* if publicAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil { - if vpcCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil { + /* if publicAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil { + if vpcCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil { cidr := strings.Split(vpcCidrBlock, "/") network.Addresses = append(network.Addresses, publicAddr+"/"+cidr[1]) } @@ -145,25 +143,3 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) { func (ms MetadataService) Type() string { return "ec2-metadata-service" } - -func (ms MetadataService) fetchAttributes(key string) ([]string, error) { - url := ms.MetadataURL() + key - resp, err := ms.FetchData(url) - if err != nil { - return nil, err - } - scanner := bufio.NewScanner(bytes.NewBuffer(resp)) - data := make([]string, 0) - for scanner.Scan() { - data = append(data, scanner.Text()) - } - return data, scanner.Err() -} - -func (ms MetadataService) fetchAttribute(key string) (string, error) { - attrs, err := ms.fetchAttributes(key) - if err == nil && len(attrs) > 0 { - return attrs[0], nil - } - return "", err -} diff --git a/config/cloudinit/datasource/metadata/ec2/metadata_test.go b/config/cloudinit/datasource/metadata/ec2/metadata_test.go index 4c266a7f..cafa93e1 100755 --- a/config/cloudinit/datasource/metadata/ec2/metadata_test.go +++ b/config/cloudinit/datasource/metadata/ec2/metadata_test.go @@ -34,114 +34,6 @@ func TestType(t *testing.T) { } } -func TestFetchAttributes(t *testing.T) { - for _, s := range []struct { - resources map[string]string - err error - tests []struct { - path string - val []string - } - }{ - { - resources: map[string]string{ - "/": "a\nb\nc/", - "/c/": "d\ne/", - "/c/e/": "f", - "/a": "1", - "/b": "2", - "/c/d": "3", - "/c/e/f": "4", - }, - tests: []struct { - path string - val []string - }{ - {"/", []string{"a", "b", "c/"}}, - {"/b", []string{"2"}}, - {"/c/d", []string{"3"}}, - {"/c/e/", []string{"f"}}, - }, - }, - { - err: fmt.Errorf("test error"), - tests: []struct { - path string - val []string - }{ - {"", nil}, - }, - }, - } { - service := MetadataService{metadata.Service{ - Client: &test.HTTPClient{Resources: s.resources, Err: s.err}, - }} - for _, tt := range s.tests { - attrs, err := service.fetchAttributes(tt.path) - if err != s.err { - t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err) - } - if !reflect.DeepEqual(attrs, tt.val) { - t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs) - } - } - } -} - -func TestFetchAttribute(t *testing.T) { - for _, s := range []struct { - resources map[string]string - err error - tests []struct { - path string - val string - } - }{ - { - resources: map[string]string{ - "/": "a\nb\nc/", - "/c/": "d\ne/", - "/c/e/": "f", - "/a": "1", - "/b": "2", - "/c/d": "3", - "/c/e/f": "4", - }, - tests: []struct { - path string - val string - }{ - {"/a", "1"}, - {"/b", "2"}, - {"/c/d", "3"}, - {"/c/e/f", "4"}, - }, - }, - { - err: fmt.Errorf("test error"), - tests: []struct { - path string - val string - }{ - {"", ""}, - }, - }, - } { - service := MetadataService{metadata.Service{ - Client: &test.HTTPClient{Resources: s.resources, Err: s.err}, - }} - for _, tt := range s.tests { - attr, err := service.fetchAttribute(tt.path) - if err != s.err { - t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err) - } - if attr != tt.val { - t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr) - } - } - } -} - func TestFetchMetadata(t *testing.T) { for _, tt := range []struct { root string diff --git a/config/cloudinit/datasource/metadata/metadata.go b/config/cloudinit/datasource/metadata/metadata.go index 37bf3944..97df4180 100755 --- a/config/cloudinit/datasource/metadata/metadata.go +++ b/config/cloudinit/datasource/metadata/metadata.go @@ -15,6 +15,8 @@ package metadata import ( + "bufio" + "bytes" "fmt" "net/http" "strings" @@ -84,3 +86,25 @@ func (ms Service) MetadataURL() string { func (ms Service) UserdataURL() string { return (ms.Root + ms.UserdataPath) } + +func (ms Service) FetchAttributes(key string) ([]string, error) { + url := ms.MetadataURL() + key + resp, err := ms.FetchData(url) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewBuffer(resp)) + data := make([]string, 0) + for scanner.Scan() { + data = append(data, scanner.Text()) + } + return data, scanner.Err() +} + +func (ms Service) FetchAttribute(key string) (string, error) { + attrs, err := ms.FetchAttributes(key) + if err == nil && len(attrs) > 0 { + return attrs[0], nil + } + return "", err +} diff --git a/config/cloudinit/datasource/metadata/metadata_test.go b/config/cloudinit/datasource/metadata/metadata_test.go index 989f1670..1ec63f70 100644 --- a/config/cloudinit/datasource/metadata/metadata_test.go +++ b/config/cloudinit/datasource/metadata/metadata_test.go @@ -17,6 +17,7 @@ package metadata import ( "bytes" "fmt" + "reflect" "testing" "github.com/rancher/os/config/cloudinit/datasource/metadata/test" @@ -177,6 +178,114 @@ func TestNewDatasource(t *testing.T) { } } +func TestFetchAttributes(t *testing.T) { + for _, s := range []struct { + resources map[string]string + err error + tests []struct { + path string + val []string + } + }{ + { + resources: map[string]string{ + "/": "a\nb\nc/", + "/c/": "d\ne/", + "/c/e/": "f", + "/a": "1", + "/b": "2", + "/c/d": "3", + "/c/e/f": "4", + }, + tests: []struct { + path string + val []string + }{ + {"/", []string{"a", "b", "c/"}}, + {"/b", []string{"2"}}, + {"/c/d", []string{"3"}}, + {"/c/e/", []string{"f"}}, + }, + }, + { + err: fmt.Errorf("test error"), + tests: []struct { + path string + val []string + }{ + {"", nil}, + }, + }, + } { + service := &Service{ + Client: &test.HTTPClient{Resources: s.resources, Err: s.err}, + } + for _, tt := range s.tests { + attrs, err := service.FetchAttributes(tt.path) + if err != s.err { + t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err) + } + if !reflect.DeepEqual(attrs, tt.val) { + t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs) + } + } + } +} + +func TestFetchAttribute(t *testing.T) { + for _, s := range []struct { + resources map[string]string + err error + tests []struct { + path string + val string + } + }{ + { + resources: map[string]string{ + "/": "a\nb\nc/", + "/c/": "d\ne/", + "/c/e/": "f", + "/a": "1", + "/b": "2", + "/c/d": "3", + "/c/e/f": "4", + }, + tests: []struct { + path string + val string + }{ + {"/a", "1"}, + {"/b", "2"}, + {"/c/d", "3"}, + {"/c/e/f", "4"}, + }, + }, + { + err: fmt.Errorf("test error"), + tests: []struct { + path string + val string + }{ + {"", ""}, + }, + }, + } { + service := &Service{ + Client: &test.HTTPClient{Resources: s.resources, Err: s.err}, + } + for _, tt := range s.tests { + attr, err := service.FetchAttribute(tt.path) + if err != s.err { + t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err) + } + if attr != tt.val { + t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr) + } + } + } +} + func Error(err error) string { if err != nil { return err.Error()