Merge pull request #24703 from hpcloud/hpe/vsphere-instance

Automatic merge from submit-queue

vSphere Cloud Provider Implementation 

This is the first PR towards implementation for vSphere cloud provider support in Kubernetes (ref. issue #23932).

pkg/cloudprovider/providers/providers.go

@@ -24,4 +24,5 @@ import (
 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt"
 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 )

pkg/cloudprovider/providers/vsphere/OWNERS

@@ -0,0 +1,4 @@
+maintainers:
+  - dagnello
+  - abithap
+

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -0,0 +1,428 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os/exec"
+	"strings"
+
+	"github.com/vmware/govmomi"
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+	"gopkg.in/gcfg.v1"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+const ProviderName = "vsphere"
+const ActivePowerState = "poweredOn"
+
+// VSphere is an implementation of cloud provider Interface for VSphere.
+type VSphere struct {
+	cfg *VSphereConfig
+	// InstanceID of the server where this VSphere object is instantiated.
+	localInstanceID string
+}
+
+type VSphereConfig struct {
+	Global struct {
+		User         string `gcfg:"user"`
+		Password     string `gcfg:"password"`
+		VCenterIP    string `gcfg:"server"`
+		VCenterPort  string `gcfg:"port"`
+		InsecureFlag bool   `gcfg:"insecure-flag"`
+		Datacenter   string `gcfg:"datacenter"`
+		Datastore    string `gcfg:"datastore"`
+	}
+
+	Network struct {
+		PublicNetwork string `gcfg:"public-network"`
+	}
+}
+
+func readConfig(config io.Reader) (VSphereConfig, error) {
+	if config == nil {
+		err := fmt.Errorf("no vSphere cloud provider config file given")
+		return VSphereConfig{}, err
+	}
+
+	var cfg VSphereConfig
+	err := gcfg.ReadInto(&cfg, config)
+	return cfg, err
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
+		cfg, err := readConfig(config)
+		if err != nil {
+			return nil, err
+		}
+		return newVSphere(cfg)
+	})
+}
+
+func readInstanceID(cfg *VSphereConfig) (string, error) {
+	cmd := exec.Command("bash", "-c", `dmidecode -t 1 | grep UUID | tr -d ' ' | cut -f 2 -d ':'`)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create vSphere client
+	c, err := vsphereLogin(cfg, ctx)
+	if err != nil {
+		return "", err
+	}
+	defer c.Logout(ctx)
+
+	// Create a new finder
+	f := find.NewFinder(c.Client, true)
+
+	// Fetch and set data center
+	dc, err := f.Datacenter(ctx, cfg.Global.Datacenter)
+	if err != nil {
+		return "", err
+	}
+	f.SetDatacenter(dc)
+
+	s := object.NewSearchIndex(c.Client)
+
+	svm, err := s.FindByUuid(ctx, dc, strings.ToLower(strings.TrimSpace(out.String())), true, nil)
+	var vm mo.VirtualMachine
+	err = s.Properties(ctx, svm.Reference(), []string{"name"}, &vm)
+	if err != nil {
+		return "", err
+	}
+	return vm.Name, nil
+}
+
+func newVSphere(cfg VSphereConfig) (*VSphere, error) {
+	id, err := readInstanceID(&cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	vs := VSphere{
+		cfg:             &cfg,
+		localInstanceID: id,
+	}
+	return &vs, nil
+}
+
+func vsphereLogin(cfg *VSphereConfig, ctx context.Context) (*govmomi.Client, error) {
+
+	// Parse URL from string
+	u, err := url.Parse(fmt.Sprintf("https://%s:%s/sdk", cfg.Global.VCenterIP, cfg.Global.VCenterPort))
+	if err != nil {
+		return nil, err
+	}
+	// set username and password for the URL
+	u.User = url.UserPassword(cfg.Global.User, cfg.Global.Password)
+
+	// Connect and log in to ESX or vCenter
+	c, err := govmomi.NewClient(ctx, u, cfg.Global.InsecureFlag)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func getVirtualMachineByName(cfg *VSphereConfig, ctx context.Context, c *govmomi.Client, name string) (*object.VirtualMachine, error) {
+	// Create a new finder
+	f := find.NewFinder(c.Client, true)
+
+	// Fetch and set data center
+	dc, err := f.Datacenter(ctx, cfg.Global.Datacenter)
+	if err != nil {
+		return nil, err
+	}
+	f.SetDatacenter(dc)
+
+	// Retrieve vm by name
+	//TODO: also look for vm inside subfolders
+	vm, err := f.VirtualMachine(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	return vm, nil
+}
+
+func getVirtualMachineManagedObjectReference(ctx context.Context, c *govmomi.Client, vm *object.VirtualMachine, field string, dst interface{}) error {
+	collector := property.DefaultCollector(c.Client)
+
+	// Retrieve required field from VM object
+	err := collector.RetrieveOne(ctx, vm.Reference(), []string{field}, dst)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getInstances(cfg *VSphereConfig, ctx context.Context, c *govmomi.Client, filter string) ([]string, error) {
+	f := find.NewFinder(c.Client, true)
+	dc, err := f.Datacenter(ctx, cfg.Global.Datacenter)
+	if err != nil {
+		return nil, err
+	}
+
+	f.SetDatacenter(dc)
+
+	//TODO: get all vms inside subfolders
+	vms, err := f.VirtualMachineList(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+
+	var vmRef []types.ManagedObjectReference
+	for _, vm := range vms {
+		vmRef = append(vmRef, vm.Reference())
+	}
+
+	pc := property.DefaultCollector(c.Client)
+
+	var vmt []mo.VirtualMachine
+	err = pc.Retrieve(ctx, vmRef, []string{"name", "summary"}, &vmt)
+	if err != nil {
+		return nil, err
+	}
+
+	var vmList []string
+	for _, vm := range vmt {
+		if vm.Summary.Runtime.PowerState == ActivePowerState {
+			vmList = append(vmList, vm.Name)
+		} else if vm.Summary.Config.Template == false {
+			glog.Warningf("VM %s, is not in %s state", vm.Name, ActivePowerState)
+		}
+	}
+	return vmList, nil
+}
+
+type Instances struct {
+	cfg             *VSphereConfig
+	localInstanceID string
+}
+
+// Instances returns an implementation of Instances for vSphere.
+func (vs *VSphere) Instances() (cloudprovider.Instances, bool) {
+	return &Instances{vs.cfg, vs.localInstanceID}, true
+}
+
+// List is an implementation of Instances.List.
+func (i *Instances) List(filter string) ([]string, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	c, err := vsphereLogin(i.cfg, ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer c.Logout(ctx)
+
+	vmList, err := getInstances(i.cfg, ctx, c, filter)
+	if err != nil {
+		return nil, err
+	}
+
+	glog.V(3).Infof("Found %s instances matching %s: %s",
+		len(vmList), filter, vmList)
+
+	return vmList, nil
+}
+
+// NodeAddresses is an implementation of Instances.NodeAddresses.
+func (i *Instances) NodeAddresses(name string) ([]api.NodeAddress, error) {
+	addrs := []api.NodeAddress{}
+
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create vSphere client
+	c, err := vsphereLogin(i.cfg, ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer c.Logout(ctx)
+
+	vm, err := getVirtualMachineByName(i.cfg, ctx, c, name)
+	if err != nil {
+		return nil, err
+	}
+
+	var mvm mo.VirtualMachine
+	err = getVirtualMachineManagedObjectReference(ctx, c, vm, "guest.net", &mvm)
+	if err != nil {
+		return nil, err
+	}
+
+	// retrieve VM's ip(s)
+	for _, v := range mvm.Guest.Net {
+		var addressType api.NodeAddressType
+		if i.cfg.Network.PublicNetwork == v.Network {
+			addressType = api.NodeExternalIP
+		} else {
+			addressType = api.NodeInternalIP
+		}
+		for _, ip := range v.IpAddress {
+			api.AddToNodeAddresses(&addrs,
+				api.NodeAddress{
+					Type:    addressType,
+					Address: ip,
+				},
+			)
+		}
+	}
+	return addrs, nil
+}
+
+func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}
+
+func (i *Instances) CurrentNodeName(hostname string) (string, error) {
+	return i.localInstanceID, nil
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (i *Instances) ExternalID(name string) (string, error) {
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create vSphere client
+	c, err := vsphereLogin(i.cfg, ctx)
+	if err != nil {
+		return "", err
+	}
+	defer c.Logout(ctx)
+
+	vm, err := getVirtualMachineByName(i.cfg, ctx, c, name)
+	if err != nil {
+		return "", err
+	}
+
+	var mvm mo.VirtualMachine
+	err = getVirtualMachineManagedObjectReference(ctx, c, vm, "summary", &mvm)
+	if err != nil {
+		return "", err
+	}
+
+	if mvm.Summary.Runtime.PowerState == ActivePowerState {
+		return vm.InventoryPath, nil
+	}
+
+	if mvm.Summary.Config.Template == false {
+		glog.Warningf("VM %s, is not in %s state", name, ActivePowerState)
+	} else {
+		glog.Warningf("VM %s, is a template", name)
+	}
+
+	return "", cloudprovider.InstanceNotFound
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+func (i *Instances) InstanceID(name string) (string, error) {
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create vSphere client
+	c, err := vsphereLogin(i.cfg, ctx)
+	if err != nil {
+		return "", err
+	}
+	defer c.Logout(ctx)
+
+	vm, err := getVirtualMachineByName(i.cfg, ctx, c, name)
+
+	var mvm mo.VirtualMachine
+	err = getVirtualMachineManagedObjectReference(ctx, c, vm, "summary", &mvm)
+	if err != nil {
+		return "", err
+	}
+
+	if mvm.Summary.Runtime.PowerState == ActivePowerState {
+		return "/" + vm.InventoryPath, nil
+	}
+
+	if mvm.Summary.Config.Template == false {
+		glog.Warning("VM %s, is not in %s state", name, ActivePowerState)
+	} else {
+		glog.Warning("VM %s, is a template", name)
+	}
+
+	return "", cloudprovider.InstanceNotFound
+}
+
+func (i *Instances) InstanceType(name string) (string, error) {
+	return "", nil
+}
+
+func (vs *VSphere) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, true
+}
+
+// ProviderName returns the cloud provider ID.
+func (vs *VSphere) ProviderName() string {
+	return ProviderName
+}
+
+// LoadBalancer returns an implementation of LoadBalancer for vSphere.
+func (vs *VSphere) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return nil, false
+}
+
+// Zones returns an implementation of Zones for Google vSphere.
+func (vs *VSphere) Zones() (cloudprovider.Zones, bool) {
+	glog.V(4).Info("Claiming to support Zones")
+
+	return vs, true
+}
+
+func (vs *VSphere) GetZone() (cloudprovider.Zone, error) {
+	glog.V(4).Infof("Current zone is %v", vs.cfg.Global.Datacenter)
+
+	return cloudprovider.Zone{Region: vs.cfg.Global.Datacenter}, nil
+}
+
+// Routes returns an implementation of Routes for vSphere.
+func (vs *VSphere) Routes() (cloudprovider.Routes, bool) {
+	return nil, true
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (vs *VSphere) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}

pkg/cloudprovider/providers/vsphere/vsphere_test.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"golang.org/x/net/context"
+)
+
+func configFromEnv() (cfg VSphereConfig, ok bool) {
+	var InsecureFlag bool
+	var err error
+	cfg.Global.VCenterIP = os.Getenv("VSPHERE_VCENTER")
+	cfg.Global.VCenterPort = os.Getenv("VSPHERE_VCENTER_PORT")
+	cfg.Global.User = os.Getenv("VSPHERE_USER")
+	cfg.Global.Password = os.Getenv("VSPHERE_PASSWORD")
+	cfg.Global.Datacenter = os.Getenv("VSPHERE_DATACENTER")
+	cfg.Network.PublicNetwork = os.Getenv("VSPHERE_PUBLIC_NETWORK")
+	if os.Getenv("VSPHERE_INSECURE") != "" {
+		InsecureFlag, err = strconv.ParseBool(os.Getenv("VSPHERE_INSECURE"))
+	} else {
+		InsecureFlag = false
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg.Global.InsecureFlag = InsecureFlag
+
+	ok = (cfg.Global.VCenterIP != "" &&
+		cfg.Global.User != "")
+
+	return
+}
+
+func TestReadConfig(t *testing.T) {
+	_, err := readConfig(nil)
+	if err == nil {
+		t.Errorf("Should fail when no config is provided: %s", err)
+	}
+
+	cfg, err := readConfig(strings.NewReader(`
+[Global]
+server = 0.0.0.0
+port = 443
+user = user
+password = password
+insecure-flag = true
+datacenter = us-west
+`))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Global.VCenterIP != "0.0.0.0" {
+		t.Errorf("incorrect vcenter ip: %s", cfg.Global.VCenterIP)
+	}
+
+	if cfg.Global.VCenterPort != "443" {
+		t.Errorf("incorrect vcenter port: %s", cfg.Global.VCenterPort)
+	}
+
+	if cfg.Global.Datacenter != "us-west" {
+		t.Errorf("incorrect datacenter: %s", cfg.Global.Datacenter)
+	}
+}
+
+func TestNewVSphere(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	_, err := newVSphere(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate vSphere: %s", err)
+	}
+}
+
+func TestVSphereLogin(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	// Create vSphere configuration object
+	vs, err := newVSphere(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate vSphere: %s", err)
+	}
+
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create vSphere client
+	c, err := vsphereLogin(vs.cfg, ctx)
+	if err != nil {
+		t.Errorf("Failed to create vSpere client: %s", err)
+	}
+	defer c.Logout(ctx)
+}
+
+func TestZones(t *testing.T) {
+	cfg := VSphereConfig{}
+	cfg.Global.Datacenter = "myDatacenter"
+
+	// Create vSphere configuration object
+	vs := VSphere{
+		cfg: &cfg,
+	}
+
+	z, ok := vs.Zones()
+	if !ok {
+		t.Fatalf("Zones() returned false")
+	}
+
+	zone, err := z.GetZone()
+	if err != nil {
+		t.Fatalf("GetZone() returned error: %s", err)
+	}
+
+	if zone.Region != vs.cfg.Global.Datacenter {
+		t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
+	}
+}
+
+func TestInstances(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	vs, err := newVSphere(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate vSphere: %s", err)
+	}
+
+	i, ok := vs.Instances()
+	if !ok {
+		t.Fatalf("Instances() returned false")
+	}
+
+	srvs, err := i.List("*")
+	if err != nil {
+		t.Fatalf("Instances.List() failed: %s", err)
+	}
+
+	if len(srvs) == 0 {
+		t.Fatalf("Instances.List() returned zero servers")
+	}
+	t.Logf("Found servers (%d): %s\n", len(srvs), srvs)
+
+	externalId, err := i.ExternalID(srvs[0])
+	if err != nil {
+		t.Fatalf("Instances.ExternalID(%s) failed: %s", srvs[0], err)
+	}
+	t.Logf("Found ExternalID(%s) = %s\n", srvs[0], externalId)
+
+	instanceId, err := i.InstanceID(srvs[0])
+	if err != nil {
+		t.Fatalf("Instances.InstanceID(%s) failed: %s", srvs[0], err)
+	}
+	t.Logf("Found InstanceID(%s) = %s\n", srvs[0], instanceId)
+
+	addrs, err := i.NodeAddresses(srvs[0])
+	if err != nil {
+		t.Fatalf("Instances.NodeAddresses(%s) failed: %s", srvs[0], err)
+	}
+	t.Logf("Found NodeAddresses(%s) = %s\n", srvs[0], addrs)
+}