vSphere test infrastructure improvement and new node-unregister test

test/e2e/storage/vsphere/BUILD

@@ -8,9 +8,15 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "bootstrap.go",
+        "config.go",
+        "connection.go",
+        "context.go",
+        "nodemapper.go",
         "persistent_volumes-vsphere.go",
         "pv_reclaimpolicy.go",
         "pvc_label_selector.go",
+        "vsphere.go",
         "vsphere_common.go",
         "vsphere_scale.go",
         "vsphere_statefulsets.go",
@@ -22,6 +28,7 @@ go_library(
         "vsphere_volume_disksize.go",
         "vsphere_volume_fstype.go",
         "vsphere_volume_master_restart.go",
+        "vsphere_volume_node_delete.go",
         "vsphere_volume_node_poweroff.go",
         "vsphere_volume_ops_storm.go",
         "vsphere_volume_perf.go",
@@ -35,10 +42,18 @@ go_library(
         "//pkg/volume/util/volumehelper:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/storage/utils:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/vmware/govmomi:go_default_library",
+        "//vendor/github.com/vmware/govmomi/find:go_default_library",
+        "//vendor/github.com/vmware/govmomi/object:go_default_library",
+        "//vendor/github.com/vmware/govmomi/session:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
         "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/storage/v1:go_default_library",
@@ -62,9 +77,6 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [
-        ":package-srcs",
-        "//test/e2e/storage/vsphere/bootstrap:all-srcs",
-    ],
+    srcs = [":package-srcs"],
     tags = ["automanaged"],
 )

test/e2e/storage/vsphere/bootstrap.go

@@ -14,18 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package bootstrap
+package vsphere
 
 import (
+	"k8s.io/kubernetes/test/e2e/framework"
 	"sync"
 )
 
 var once sync.Once
 var waiting = make(chan bool)
+var f *framework.Framework
 
 // Bootstrap takes care of initializing necessary test context for vSphere tests
-func Bootstrap() {
+func Bootstrap(fw *framework.Framework) {
 	done := make(chan bool)
+	f = fw
 	go func() {
 		once.Do(bootstrapOnce)
 		<-waiting
@@ -35,10 +38,19 @@ func Bootstrap() {
 }
 
 func bootstrapOnce() {
-	// TBD
 	// 1. Read vSphere conf and get VSphere instances
-	// 2. Get Node to VSphere mapping
-	// 3. Set NodeMapper in vSphere context
-	TestContext = Context{}
+	vsphereInstances, err := GetVSphereInstances()
+	if err != nil {
+		framework.Failf("Failed to bootstrap vSphere with error: %v", err)
+	}
+	// 2. Get all ready nodes
+	nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+	TestContext = VSphereContext{NodeMapper: &NodeMapper{}, VSphereInstances: vsphereInstances}
+
+	// 3. Get Node to VSphere mapping
+	err = TestContext.NodeMapper.GenerateNodeMap(vsphereInstances, *nodeList)
+	if err != nil {
+		framework.Failf("Failed to bootstrap vSphere with error: %v", err)
+	}
 	close(waiting)
 }

test/e2e/storage/vsphere/bootstrap/BUILD

@@ -1,25 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "bootstrap.go",
-        "context.go",
-    ],
-    importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere/bootstrap",
-    visibility = ["//visibility:public"],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-    visibility = ["//visibility:public"],
-)

test/e2e/storage/vsphere/config.go

@@ -0,0 +1,178 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"errors"
+	"fmt"
+	"gopkg.in/gcfg.v1"
+	"io"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"os"
+)
+
+const (
+	vSphereConfFileEnvVar = "VSPHERE_CONF_FILE"
+)
+
+var (
+	confFileLocation = os.Getenv(vSphereConfFileEnvVar)
+)
+
+// Config represents vSphere configuration
+type Config struct {
+	Username          string
+	Password          string
+	Hostname          string
+	Port              string
+	Datacenters       string
+	RoundTripperCount uint
+	DefaultDatastore  string
+}
+
+// ConfigFile represents the content of vsphere.conf file.
+// Users specify the configuration of one or more vSphere instances in vsphere.conf where
+// the Kubernetes master and worker nodes are running.
+type ConfigFile struct {
+	Global struct {
+		// vCenter username.
+		User string `gcfg:"user"`
+		// vCenter password in clear text.
+		Password string `gcfg:"password"`
+		// vCenter port.
+		VCenterPort string `gcfg:"port"`
+		// True if vCenter uses self-signed cert.
+		InsecureFlag bool `gcfg:"insecure-flag"`
+		// Datacenter in which VMs are located.
+		Datacenters string `gcfg:"datacenters"`
+		// Soap round tripper count (retries = RoundTripper - 1)
+		RoundTripperCount uint `gcfg:"soap-roundtrip-count"`
+	}
+
+	VirtualCenter map[string]*Config
+
+	Network struct {
+		// PublicNetwork is name of the network the VMs are joined to.
+		PublicNetwork string `gcfg:"public-network"`
+	}
+
+	Disk struct {
+		// SCSIControllerType defines SCSI controller to be used.
+		SCSIControllerType string `dcfg:"scsicontrollertype"`
+	}
+
+	// Endpoint used to create volumes
+	Workspace struct {
+		VCenterIP        string `gcfg:"server"`
+		Datacenter       string `gcfg:"datacenter"`
+		Folder           string `gcfg:"folder"`
+		DefaultDatastore string `gcfg:"default-datastore"`
+		ResourcePoolPath string `gcfg:"resourcepool-path"`
+	}
+}
+
+// GetVSphereInstances parses vsphere.conf and returns VSphere instances
+func GetVSphereInstances() (map[string]*VSphere, error) {
+	cfg, err := getConfig()
+	if err != nil {
+		return nil, err
+	}
+	return populateInstanceMap(cfg)
+}
+
+func getConfig() (*ConfigFile, error) {
+	if confFileLocation == "" {
+		return nil, fmt.Errorf("Env variable 'VSPHERE_CONF_FILE' is not set.")
+	}
+	confFile, err := os.Open(confFileLocation)
+	if err != nil {
+		return nil, err
+	}
+	defer confFile.Close()
+	cfg, err := readConfig(confFile)
+	if err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+// readConfig parses vSphere cloud config file into ConfigFile.
+func readConfig(config io.Reader) (ConfigFile, error) {
+	if config == nil {
+		err := fmt.Errorf("no vSphere cloud provider config file given")
+		return ConfigFile{}, err
+	}
+
+	var cfg ConfigFile
+	err := gcfg.ReadInto(&cfg, config)
+	return cfg, err
+}
+
+func populateInstanceMap(cfg *ConfigFile) (map[string]*VSphere, error) {
+	vsphereInstances := make(map[string]*VSphere)
+
+	if cfg.Workspace.VCenterIP == "" || cfg.Workspace.DefaultDatastore == "" || cfg.Workspace.Folder == "" || cfg.Workspace.Datacenter == "" {
+		msg := fmt.Sprintf("All fields in workspace are mandatory."+
+			" vsphere.conf does not have the workspace specified correctly. cfg.Workspace: %+v", cfg.Workspace)
+		framework.Logf(msg)
+		return nil, errors.New(msg)
+	}
+	for vcServer, vcConfig := range cfg.VirtualCenter {
+		framework.Logf("Initializing vc server %s", vcServer)
+		if vcServer == "" {
+			framework.Logf("vsphere.conf does not have the VirtualCenter IP address specified")
+			return nil, errors.New("vsphere.conf does not have the VirtualCenter IP address specified")
+		}
+		vcConfig.Hostname = vcServer
+
+		if vcConfig.Username == "" {
+			vcConfig.Username = cfg.Global.User
+		}
+		if vcConfig.Password == "" {
+			vcConfig.Password = cfg.Global.Password
+		}
+		if vcConfig.Username == "" {
+			msg := fmt.Sprintf("vcConfig.User is empty for vc %s!", vcServer)
+			framework.Logf(msg)
+			return nil, errors.New(msg)
+		}
+		if vcConfig.Password == "" {
+			msg := fmt.Sprintf("vcConfig.Password is empty for vc %s!", vcServer)
+			framework.Logf(msg)
+			return nil, errors.New(msg)
+		}
+		if vcConfig.Port == "" {
+			vcConfig.Port = cfg.Global.VCenterPort
+		}
+		if vcConfig.Datacenters == "" && cfg.Global.Datacenters != "" {
+			vcConfig.Datacenters = cfg.Global.Datacenters
+		}
+		if vcConfig.RoundTripperCount == 0 {
+			vcConfig.RoundTripperCount = cfg.Global.RoundTripperCount
+		}
+
+		vcConfig.DefaultDatastore = cfg.Workspace.DefaultDatastore
+
+		vsphereIns := VSphere{
+			Config: vcConfig,
+		}
+		vsphereInstances[vcServer] = &vsphereIns
+	}
+
+	framework.Logf("ConfigFile %v \n vSphere instances %v", cfg, vsphereInstances)
+	return vsphereInstances, nil
+}

test/e2e/storage/vsphere/connection.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"fmt"
+	neturl "net/url"
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi"
+	"github.com/vmware/govmomi/session"
+	"github.com/vmware/govmomi/vim25"
+	"golang.org/x/net/context"
+)
+
+const (
+	roundTripperDefaultCount = 3
+)
+
+var (
+	clientLock sync.Mutex
+)
+
+// Connect makes connection to vSphere
+// No actions are taken if a connection exists and alive. Otherwise, a new client will be created.
+func Connect(ctx context.Context, vs *VSphere) error {
+	var err error
+	clientLock.Lock()
+	defer clientLock.Unlock()
+
+	if vs.Client == nil {
+		vs.Client, err = NewClient(ctx, vs)
+		if err != nil {
+			glog.Errorf("Failed to create govmomi client. err: %+v", err)
+			return err
+		}
+		return nil
+	}
+	manager := session.NewManager(vs.Client.Client)
+	userSession, err := manager.UserSession(ctx)
+	if err != nil {
+		glog.Errorf("Error while obtaining user session. err: %+v", err)
+		return err
+	}
+	if userSession != nil {
+		return nil
+	}
+	glog.Warningf("Creating new client session since the existing session is not valid or not authenticated")
+	vs.Client.Logout(ctx)
+	vs.Client, err = NewClient(ctx, vs)
+	if err != nil {
+		glog.Errorf("Failed to create govmomi client. err: %+v", err)
+		return err
+	}
+	return nil
+}
+
+// NewClient creates a new client for vSphere connection
+func NewClient(ctx context.Context, vs *VSphere) (*govmomi.Client, error) {
+	url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", vs.Config.Hostname, vs.Config.Port))
+	if err != nil {
+		glog.Errorf("Failed to parse URL: %s. err: %+v", url, err)
+		return nil, err
+	}
+	url.User = neturl.UserPassword(vs.Config.Username, vs.Config.Password)
+	client, err := govmomi.NewClient(ctx, url, true)
+	if err != nil {
+		glog.Errorf("Failed to create new client. err: %+v", err)
+		return nil, err
+	}
+	if vs.Config.RoundTripperCount == 0 {
+		vs.Config.RoundTripperCount = roundTripperDefaultCount
+	}
+	client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(int(vs.Config.RoundTripperCount)))
+	return client, nil
+}

test/e2e/storage/vsphere/context.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package bootstrap
+package vsphere
 
 // Context holds common information for vSphere tests
-type Context struct {
-	// NodeMapper and other instances, common to vSphere tests
+type VSphereContext struct {
+	NodeMapper       *NodeMapper
+	VSphereInstances map[string]*VSphere
 }
 
 // TestContext should be used by all tests to access common context data. It should be initialized only once, during bootstrapping the tests.
-var TestContext Context
+var TestContext VSphereContext

test/e2e/storage/vsphere/nodemapper.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"errors"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"strings"
+	"sync"
+)
+
+type NodeMapper struct {
+}
+
+type NodeInfo struct {
+	Name              string
+	DataCenterRef     types.ManagedObjectReference
+	VirtualMachineRef types.ManagedObjectReference
+	VSphere           *VSphere
+}
+
+var (
+	nameToNodeInfo = make(map[string]*NodeInfo)
+)
+
+// GenerateNodeMap populates node name to node info map
+func (nm *NodeMapper) GenerateNodeMap(vSphereInstances map[string]*VSphere, nodeList v1.NodeList) error {
+	type VmSearch struct {
+		vs         *VSphere
+		datacenter *object.Datacenter
+	}
+
+	var wg sync.WaitGroup
+	var queueChannel []*VmSearch
+
+	var datacenters []*object.Datacenter
+	var err error
+	for _, vs := range vSphereInstances {
+
+		// Create context
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		if vs.Config.Datacenters == "" {
+			datacenters, err = vs.GetAllDatacenter(ctx)
+			if err != nil {
+				framework.Logf("NodeMapper error: %v", err)
+				continue
+			}
+		} else {
+			dcName := strings.Split(vs.Config.Datacenters, ",")
+			for _, dc := range dcName {
+				dc = strings.TrimSpace(dc)
+				if dc == "" {
+					continue
+				}
+				datacenter, err := vs.GetDatacenter(ctx, dc)
+				if err != nil {
+					framework.Logf("NodeMapper error dc: %s \n err: %v", dc, err)
+
+					continue
+				}
+				datacenters = append(datacenters, datacenter)
+			}
+		}
+
+		for _, dc := range datacenters {
+			framework.Logf("Search candidates vc=%s and datacenter=%s", vs.Config.Hostname, dc.Name())
+			queueChannel = append(queueChannel, &VmSearch{vs: vs, datacenter: dc})
+		}
+	}
+
+	for _, node := range nodeList.Items {
+		n := node
+
+		go func() {
+			nodeUUID := n.Status.NodeInfo.SystemUUID
+			framework.Logf("Searching for node with UUID: %s", nodeUUID)
+			for _, res := range queueChannel {
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+				vm, err := res.vs.GetVMByUUID(ctx, nodeUUID, res.datacenter)
+				if err != nil {
+					framework.Logf("Error %v while looking for node=%s in vc=%s and datacenter=%s",
+						err, n.Name, res.vs.Config.Hostname, res.datacenter.Name())
+					continue
+				}
+				if vm != nil {
+					framework.Logf("Found node %s as vm=%+v in vc=%s and datacenter=%s",
+						n.Name, vm, res.vs.Config.Hostname, res.datacenter.Name())
+					nodeInfo := &NodeInfo{Name: n.Name, DataCenterRef: res.datacenter.Reference(), VirtualMachineRef: vm.Reference(), VSphere: res.vs}
+					nameToNodeInfo[n.Name] = nodeInfo
+					break
+				}
+			}
+			wg.Done()
+		}()
+		wg.Add(1)
+	}
+	wg.Wait()
+
+	if len(nameToNodeInfo) != len(nodeList.Items) {
+		return errors.New("all nodes not mapped to respective vSphere")
+	}
+	return nil
+}
+
+// GetNodeInfo return NodeInfo for given nodeName
+func (nm *NodeMapper) GetNodeInfo(nodeName string) *NodeInfo {
+	return nameToNodeInfo[nodeName]
+}
+
+// SetNodeInfo sets NodeInfo for given nodeName. This function is not thread safe. Users need to handle concurrency.
+func (nm *NodeMapper) SetNodeInfo(nodeName string, nodeInfo *NodeInfo) {
+	nameToNodeInfo[nodeName] = nodeInfo
+}

test/e2e/storage/vsphere/vsphere.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"github.com/vmware/govmomi"
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/object"
+	"golang.org/x/net/context"
+	"strings"
+)
+
+// Represents a vSphere instance where one or more kubernetes nodes are running.
+type VSphere struct {
+	Config *Config
+	Client *govmomi.Client
+}
+
+// GetDatacenter returns the DataCenter Object for the given datacenterPath
+func (vs *VSphere) GetDatacenter(ctx context.Context, datacenterPath string) (*object.Datacenter, error) {
+	Connect(ctx, vs)
+	finder := find.NewFinder(vs.Client.Client, true)
+	return finder.Datacenter(ctx, datacenterPath)
+}
+
+// GetAllDatacenter returns all the DataCenter Objects
+func (vs *VSphere) GetAllDatacenter(ctx context.Context) ([]*object.Datacenter, error) {
+	Connect(ctx, vs)
+	finder := find.NewFinder(vs.Client.Client, true)
+	return finder.DatacenterList(ctx, "*")
+}
+
+// GetVMByUUID gets the VM object from the given vmUUID
+func (vs *VSphere) GetVMByUUID(ctx context.Context, vmUUID string, dc object.Reference) (object.Reference, error) {
+	Connect(ctx, vs)
+	datacenter := object.NewDatacenter(vs.Client.Client, dc.Reference())
+	s := object.NewSearchIndex(vs.Client.Client)
+	vmUUID = strings.ToLower(strings.TrimSpace(vmUUID))
+	return s.FindByUuid(ctx, datacenter, vmUUID, true, nil)
+}

test/e2e/storage/vsphere/vsphere_utils.go

@@ -23,6 +23,8 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/mo"
 	"k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -36,6 +38,11 @@ import (
 	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
+
+	"context"
+
+	"github.com/vmware/govmomi/find"
+	vimtypes "github.com/vmware/govmomi/vim25/types"
 )
 
 const (
@@ -467,3 +474,96 @@ func getVSphere(c clientset.Interface) (*vsphere.VSphere, error) {
 func GetVSphere(c clientset.Interface) (*vsphere.VSphere, error) {
 	return getVSphere(c)
 }
+
+// get .vmx file path for a virtual machine
+func getVMXFilePath(vmObject *object.VirtualMachine) (vmxPath string) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var nodeVM mo.VirtualMachine
+	err := vmObject.Properties(ctx, vmObject.Reference(), []string{"config.files"}, &nodeVM)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(nodeVM.Config).NotTo(BeNil())
+
+	vmxPath = nodeVM.Config.Files.VmPathName
+	framework.Logf("vmx file path is %s", vmxPath)
+	return vmxPath
+}
+
+// verify ready node count. Try upto 3 minutes. Return true if count is expected count
+func verifyReadyNodeCount(client clientset.Interface, expectedNodes int) bool {
+	numNodes := 0
+	for i := 0; i < 36; i++ {
+		nodeList := framework.GetReadySchedulableNodesOrDie(client)
+		Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node")
+
+		numNodes = len(nodeList.Items)
+		if numNodes == expectedNodes {
+			break
+		}
+		time.Sleep(5 * time.Second)
+	}
+	return (numNodes == expectedNodes)
+}
+
+// poweroff nodeVM and confirm the poweroff state
+func poweroffNodeVM(nodeName string, vm *object.VirtualMachine) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	framework.Logf("Powering off node VM %s", nodeName)
+
+	_, err := vm.PowerOff(ctx)
+	Expect(err).NotTo(HaveOccurred())
+	err = vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOff)
+	Expect(err).NotTo(HaveOccurred(), "Unable to power off the node")
+}
+
+// poweron nodeVM and confirm the poweron state
+func poweronNodeVM(nodeName string, vm *object.VirtualMachine) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	framework.Logf("Powering on node VM %s", nodeName)
+
+	vm.PowerOn(ctx)
+	err := vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOn)
+	Expect(err).NotTo(HaveOccurred(), "Unable to power on the node")
+}
+
+// unregister a nodeVM from VC
+func unregisterNodeVM(nodeName string, vm *object.VirtualMachine) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	poweroffNodeVM(nodeName, vm)
+
+	framework.Logf("Unregistering node VM %s", nodeName)
+	err := vm.Unregister(ctx)
+	Expect(err).NotTo(HaveOccurred(), "Unable to unregister the node")
+}
+
+// register a nodeVM into a VC
+func registerNodeVM(nodeName, workingDir, vmxFilePath string, rpool *object.ResourcePool, host *object.HostSystem) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	framework.Logf("Registering node VM %s with vmx file path %s", nodeName, vmxFilePath)
+
+	nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeName)
+	finder := find.NewFinder(nodeInfo.VSphere.Client.Client, true)
+
+	vmFolder, err := finder.FolderOrDefault(ctx, workingDir)
+	Expect(err).NotTo(HaveOccurred())
+
+	registerTask, err := vmFolder.RegisterVM(ctx, vmxFilePath, nodeName, false, rpool, host)
+	Expect(err).NotTo(HaveOccurred())
+	err = registerTask.Wait(ctx)
+	Expect(err).NotTo(HaveOccurred())
+
+	vmPath := filepath.Join(workingDir, nodeName)
+	vm, err := finder.VirtualMachine(ctx, vmPath)
+	Expect(err).NotTo(HaveOccurred())
+
+	poweronNodeVM(nodeName, vm)
+}

test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go

@@ -51,6 +51,7 @@ var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:v
 
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		scParameters = make(map[string]string)

test/e2e/storage/vsphere/vsphere_volume_datastore.go

@@ -54,6 +54,7 @@ var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]",
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		scParameters = make(map[string]string)

test/e2e/storage/vsphere/vsphere_volume_diskformat.go

@@ -65,6 +65,7 @@ var _ = utils.SIGDescribe("Volume Disk Format [Feature:vsphere]", func() {
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)

test/e2e/storage/vsphere/vsphere_volume_disksize.go

@@ -54,6 +54,7 @@ var _ = utils.SIGDescribe("Volume Disk Size [Feature:vsphere]", func() {
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		scParameters = make(map[string]string)

test/e2e/storage/vsphere/vsphere_volume_fstype.go

@@ -72,6 +72,7 @@ var _ = utils.SIGDescribe("Volume FStype [Feature:vsphere]", func() {
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)

test/e2e/storage/vsphere/vsphere_volume_master_restart.go

@@ -57,6 +57,7 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout))

test/e2e/storage/vsphere/vsphere_volume_node_delete.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vmware/govmomi/object"
+	"golang.org/x/net/context"
+
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+var _ = utils.SIGDescribe("Node Unregister [Feature:vsphere] [Slow] [Disruptive]", func() {
+	f := framework.NewDefaultFramework("node-unregister")
+	var (
+		client     clientset.Interface
+		namespace  string
+		workingDir string
+		err        error
+	)
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout))
+		Expect(err).NotTo(HaveOccurred())
+		workingDir = os.Getenv("VSPHERE_WORKING_DIR")
+		Expect(workingDir).NotTo(BeEmpty())
+
+	})
+
+	It("node unregister", func() {
+		By("Get total Ready nodes")
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		Expect(len(nodeList.Items) > 1).To(BeTrue(), "At least 2 nodes are required for this test")
+
+		totalNodesCount := len(nodeList.Items)
+		nodeVM := nodeList.Items[0]
+
+		nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeVM.ObjectMeta.Name)
+		vmObject := object.NewVirtualMachine(nodeInfo.VSphere.Client.Client, nodeInfo.VirtualMachineRef)
+
+		// Find VM .vmx file path, host, resource pool.
+		// They are required to register a node VM to VC
+		vmxFilePath := getVMXFilePath(vmObject)
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		vmHost, err := vmObject.HostSystem(ctx)
+		Expect(err).NotTo(HaveOccurred())
+
+		vmPool, err := vmObject.ResourcePool(ctx)
+		Expect(err).NotTo(HaveOccurred())
+
+		// Unregister Node VM
+		By("Unregister a node VM")
+		unregisterNodeVM(nodeVM.ObjectMeta.Name, vmObject)
+
+		// Ready nodes should be 1 less
+		By("Verifying the ready node counts")
+		Expect(verifyReadyNodeCount(f.ClientSet, totalNodesCount-1)).To(BeTrue(), "Unable to verify expected ready node count")
+
+		nodeList = framework.GetReadySchedulableNodesOrDie(client)
+		Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node")
+
+		var nodeNameList []string
+		for _, node := range nodeList.Items {
+			nodeNameList = append(nodeNameList, node.ObjectMeta.Name)
+		}
+		Expect(nodeNameList).NotTo(ContainElement(nodeVM.ObjectMeta.Name))
+
+		// Register Node VM
+		By("Register back the node VM")
+		registerNodeVM(nodeVM.ObjectMeta.Name, workingDir, vmxFilePath, vmPool, vmHost)
+
+		// Ready nodes should be equal to earlier count
+		By("Verifying the ready node counts")
+		Expect(verifyReadyNodeCount(f.ClientSet, totalNodesCount)).To(BeTrue(), "Unable to verify expected ready node count")
+
+		nodeList = framework.GetReadySchedulableNodesOrDie(client)
+		Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node")
+
+		nodeNameList = nodeNameList[:0]
+		for _, node := range nodeList.Items {
+			nodeNameList = append(nodeNameList, node.ObjectMeta.Name)
+		}
+		Expect(nodeNameList).To(ContainElement(nodeVM.ObjectMeta.Name))
+
+		// Sanity test that pod provisioning works
+		By("Sanity check for volume lifecycle")
+		scParameters := make(map[string]string)
+		storagePolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY")
+		Expect(storagePolicy).NotTo(BeEmpty(), "Please set VSPHERE_SPBM_GOLD_POLICY system environment")
+		scParameters[SpbmStoragePolicy] = storagePolicy
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+})

test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go

@@ -52,6 +52,7 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]",
 
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout))

test/e2e/storage/vsphere/vsphere_volume_ops_storm.go

@@ -63,6 +63,7 @@ var _ = utils.SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() {
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)

test/e2e/storage/vsphere/vsphere_volume_perf.go

@@ -63,6 +63,7 @@ var _ = utils.SIGDescribe("vcp-performance [Feature:vsphere]", func() {
 
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 

test/e2e/storage/vsphere/vsphere_volume_placement.go

@@ -52,6 +52,7 @@ var _ = utils.SIGDescribe("Volume Placement", func() {
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		c = f.ClientSet
 		ns = f.Namespace.Name
 		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))

test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go

@@ -98,6 +98,7 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp
 	)
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
 		client = f.ClientSet
 		namespace = f.Namespace.Name
 		policyName = GetAndExpectStringEnvVar(SPBMPolicyName)