From 8df21b84a11a9ccfac48c0cdf656d933bcae260b Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Wed, 6 Aug 2014 13:15:14 -0400 Subject: [PATCH] Add vagrant cloudprovider --- cluster/saltbase/salt/apiserver/default | 5 + cluster/vagrant/provision-master.sh | 24 ++- cmd/apiserver/apiserver.go | 6 + pkg/cloudprovider/vagrant.go | 188 ++++++++++++++++++++++++ pkg/cloudprovider/vagrant_test.go | 88 +++++++++++ 5 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 pkg/cloudprovider/vagrant.go create mode 100644 pkg/cloudprovider/vagrant_test.go diff --git a/cluster/saltbase/salt/apiserver/default b/cluster/saltbase/salt/apiserver/default index 61cde0d7c4d..8b4951d220e 100644 --- a/cluster/saltbase/salt/apiserver/default +++ b/cluster/saltbase/salt/apiserver/default @@ -3,7 +3,12 @@ {% set daemon_args = "" %} {% endif %} +{% set machines = ""%} {% set cloud_provider = "" %} +{% if grains.cloud_provider is defined %} + {% set cloud_provider = "-cloud_provider=" + grains.cloud_provider %} +{% endif %} + {% set minion_regexp = "-minion_regexp '" + pillar['instance_prefix'] + ".*'" %} {% set address = "-address 127.0.0.1" %} diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index bcd28bfb062..b89c07c0f0a 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -26,7 +26,7 @@ cat </etc/salt/minion.d/grains.conf grains: master_ip: $MASTER_IP etcd_servers: $MASTER_IP - minion_ips: $MINION_IPS + cloud_provider: vagrant roles: - kubernetes-master EOF @@ -63,6 +63,20 @@ echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd # we will run provision to update code each time we test, so we do not want to do salt install each time if [ ! $(which salt-master) ]; then + # Configure the salt-api + cat </etc/salt/master.d/salt-api.conf +# Set vagrant user as REST API user +external_auth: + pam: + vagrant: + - .* +rest_cherrypy: + port: 8000 + host: 127.0.0.1 + disable_ssl: True + webhook_disable_auth: True +EOF + # Install Salt # # We specify -X to avoid a race condition that can cause minion failure to @@ -70,6 +84,14 @@ if [ ! $(which salt-master) ]; then # # -M installs the master curl -sS -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M + + # Install salt-api + # + # This is used to inform the cloud provider used in the vagrant cluster + yum install -y salt-api + systemctl enable salt-api + systemctl start salt-api + fi # Build release diff --git a/cmd/apiserver/apiserver.go b/cmd/apiserver/apiserver.go index 07741bd6d7e..85e0b3cdba3 100644 --- a/cmd/apiserver/apiserver.go +++ b/cmd/apiserver/apiserver.go @@ -83,6 +83,12 @@ func main() { if err != nil { glog.Fatalf("Couldn't connect to GCE cloud: %#v", err) } + case "vagrant": + var err error + cloud, err = cloudprovider.NewVagrantCloud() + if err != nil { + glog.Fatalf("Couldn't connect to vagrant cloud: %#v", err) + } default: if len(*cloudProvider) > 0 { glog.Infof("Unknown cloud provider: %s", *cloudProvider) diff --git a/pkg/cloudprovider/vagrant.go b/pkg/cloudprovider/vagrant.go new file mode 100644 index 00000000000..6488776a3ac --- /dev/null +++ b/pkg/cloudprovider/vagrant.go @@ -0,0 +1,188 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net" + "net/http" + neturl "net/url" + "sort" +) + +// VagrantCloud is an implementation of Interface, TCPLoadBalancer and Instances for developer managed Vagrant cluster +type VagrantCloud struct { + saltURL string + saltUser string + saltPass string + saltAuth string +} + +// SaltToken is an authorization token required by Salt REST API +type SaltToken struct { + Token string `json:"token"` + User string `json:"user"` + EAuth string `json:"eauth"` +} + +// SaltLoginResponse is the response object for a /login operation against Salt REST API +type SaltLoginResponse struct { + Data []SaltToken `json:"return"` +} + +// SaltMinion is a machine managed by the Salt service +type SaltMinion struct { + Roles []string `json:"roles"` + IP string `json:"minion_ip"` + Host string `json:"host"` +} + +// SaltMinions is a map of minion name to machine information +type SaltMinions map[string]SaltMinion + +// SaltMinionsResponse is the response object for a /minions operation against Salt REST API +type SaltMinionsResponse struct { + Minions []SaltMinions `json:"return"` +} + +// NewVagrantCloud creates a new instance of VagrantCloud configured to talk to the Salt REST API. +func NewVagrantCloud() (*VagrantCloud, error) { + return &VagrantCloud{ + saltURL: "http://127.0.0.1:8000", + saltUser: "vagrant", + saltPass: "vagrant", + saltAuth: "pam", + }, nil +} + +// TCPLoadBalancer returns an implementation of TCPLoadBalancer for Vagrant cloud +func (v *VagrantCloud) TCPLoadBalancer() (TCPLoadBalancer, bool) { + return nil, false +} + +// Instances returns an implementation of Instances for Vagrant cloud +func (v *VagrantCloud) Instances() (Instances, bool) { + return v, true +} + +// Zones returns an implementation of Zones for Vagrant cloud +func (v *VagrantCloud) Zones() (Zones, bool) { + return nil, false +} + +// IPAddress returns the address of a particular machine instance +func (v *VagrantCloud) IPAddress(instance string) (net.IP, error) { + // since the instance now is the IP in the vagrant env, this is trivial no-op + return net.ParseIP(instance), nil +} + +// saltMinionsByRole filters a list of minions that have a matching role +func (v *VagrantCloud) saltMinionsByRole(minions []SaltMinion, role string) []SaltMinion { + var filteredMinions []SaltMinion + for _, value := range minions { + sort.Strings(value.Roles) + if pos := sort.SearchStrings(value.Roles, role); pos < len(value.Roles) { + filteredMinions = append(filteredMinions, value) + } + } + return filteredMinions +} + +// saltMinions invokes the Salt API for minions using provided token +func (v *VagrantCloud) saltMinions(token SaltToken) ([]SaltMinion, error) { + var minions []SaltMinion + + url := v.saltURL + "/minions" + req, err := http.NewRequest("GET", url, nil) + req.Header.Add("X-Auth-Token", token.Token) + + client := &http.Client{} + resp, err := client.Do(req) + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return minions, err + } + + var minionsResp SaltMinionsResponse + if err = json.Unmarshal(body, &minionsResp); err != nil { + return minions, err + } + + for _, value := range minionsResp.Minions[0] { + minions = append(minions, value) + } + + return minions, nil +} + +// saltLogin invokes the Salt API to get an authorization token +func (v *VagrantCloud) saltLogin() (SaltToken, error) { + url := v.saltURL + "/login" + data := neturl.Values{ + "username": {v.saltUser}, + "password": {v.saltPass}, + "eauth": {v.saltAuth}, + } + + var token SaltToken + resp, err := http.PostForm(url, data) + if err != nil { + return token, err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return token, err + } + + var loginResp SaltLoginResponse + if err := json.Unmarshal(body, &loginResp); err != nil { + return token, err + } + + if len(loginResp.Data) == 0 { + return token, errors.New("No token found in response") + } + + return loginResp.Data[0], nil +} + +// List enumerates the set of minions instances known by the cloud provider +func (v *VagrantCloud) List(filter string) ([]string, error) { + token, err := v.saltLogin() + if err != nil { + return nil, err + } + + minions, err := v.saltMinions(token) + if err != nil { + return nil, err + } + + filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool") + var instances []string + for _, instance := range filteredMinions { + instances = append(instances, instance.IP) + } + + return instances, nil +} diff --git a/pkg/cloudprovider/vagrant_test.go b/pkg/cloudprovider/vagrant_test.go new file mode 100644 index 00000000000..0877f678379 --- /dev/null +++ b/pkg/cloudprovider/vagrant_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// startSaltTestServer starts a test server that mocks the Salt REST API +func startSaltTestServer() *httptest.Server { + + // mock responses + var ( + testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "minion_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`) + testSaltLoginResponse = []byte(`{ "return": [{"perms": [".*"], "start": 1407355696.564397, "token": "ca74fa1c48ce40e204a1e820d2fa14b7cf033137", "expire": 1407398896.564398, "user": "vagrant", "eauth": "pam"}]}`) + testSaltFailure = []byte(`failure`) + ) + + handler := func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "GET": + switch r.URL.Path { + case "/minions": + w.Write(testSaltMinionsResponse) + return + } + case "POST": + switch r.URL.Path { + case "/login": + w.Write(testSaltLoginResponse) + return + } + } + w.Write(testSaltFailure) + } + return httptest.NewServer(http.HandlerFunc(handler)) +} + +// TestVagrantCloud tests against a mock Salt REST API to validate its cloud provider features +func TestVagrantCloud(t *testing.T) { + server := startSaltTestServer() + defer server.Close() + + vagrantCloud := &VagrantCloud{ + saltURL: server.URL, + saltUser: "vagrant", + saltPass: "vagrant", + saltAuth: "pam", + } + + instances, err := vagrantCloud.List("") + if err != nil { + t.Fatalf("There was an error listing instances %s", err) + } + + if len(instances) != 1 { + t.Fatalf("Incorrect number of instances returned") + } + + if instances[0] != "10.245.2.2" { + t.Fatalf("Invalid instance returned") + } + + ip, err := vagrantCloud.IPAddress(instances[0]) + if err != nil { + t.Fatalf("Unexpected error, should have returned a valid IP address: %s", err) + } + + if ip.String() != "10.245.2.2" { + t.Fatalf("Invalid IP address returned") + } +}