From 819b97ba7e1bfb3489777850c248d38dc5b5c163 Mon Sep 17 00:00:00 2001 From: prashima Date: Wed, 31 Jan 2018 21:14:55 -0800 Subject: [PATCH] vSphere test infrastructure improvement and new node-unregister test --- test/e2e/storage/vsphere/BUILD | 20 +- .../vsphere/{bootstrap => }/bootstrap.go | 24 ++- test/e2e/storage/vsphere/bootstrap/BUILD | 25 --- test/e2e/storage/vsphere/config.go | 178 ++++++++++++++++++ test/e2e/storage/vsphere/connection.go | 91 +++++++++ .../vsphere/{bootstrap => }/context.go | 9 +- test/e2e/storage/vsphere/nodemapper.go | 134 +++++++++++++ test/e2e/storage/vsphere/vsphere.go | 54 ++++++ test/e2e/storage/vsphere/vsphere_utils.go | 100 ++++++++++ .../vsphere/vsphere_volume_cluster_ds.go | 1 + .../vsphere/vsphere_volume_datastore.go | 1 + .../vsphere/vsphere_volume_diskformat.go | 1 + .../vsphere/vsphere_volume_disksize.go | 1 + .../storage/vsphere/vsphere_volume_fstype.go | 1 + .../vsphere/vsphere_volume_master_restart.go | 1 + .../vsphere/vsphere_volume_node_delete.go | 119 ++++++++++++ .../vsphere/vsphere_volume_node_poweroff.go | 1 + .../vsphere/vsphere_volume_ops_storm.go | 1 + .../storage/vsphere/vsphere_volume_perf.go | 1 + .../vsphere/vsphere_volume_placement.go | 1 + .../vsphere/vsphere_volume_vsan_policy.go | 1 + 21 files changed, 726 insertions(+), 39 deletions(-) rename test/e2e/storage/vsphere/{bootstrap => }/bootstrap.go (58%) delete mode 100644 test/e2e/storage/vsphere/bootstrap/BUILD create mode 100644 test/e2e/storage/vsphere/config.go create mode 100644 test/e2e/storage/vsphere/connection.go rename test/e2e/storage/vsphere/{bootstrap => }/context.go (84%) create mode 100644 test/e2e/storage/vsphere/nodemapper.go create mode 100644 test/e2e/storage/vsphere/vsphere.go create mode 100644 test/e2e/storage/vsphere/vsphere_volume_node_delete.go diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD index b28a862ca63..7ea0b8a7302 100644 --- a/test/e2e/storage/vsphere/BUILD +++ b/test/e2e/storage/vsphere/BUILD @@ -8,9 +8,15 @@ load( go_library( name = "go_default_library", srcs = [ + "bootstrap.go", + "config.go", + "connection.go", + "context.go", + "nodemapper.go", "persistent_volumes-vsphere.go", "pv_reclaimpolicy.go", "pvc_label_selector.go", + "vsphere.go", "vsphere_common.go", "vsphere_scale.go", "vsphere_statefulsets.go", @@ -22,6 +28,7 @@ go_library( "vsphere_volume_disksize.go", "vsphere_volume_fstype.go", "vsphere_volume_master_restart.go", + "vsphere_volume_node_delete.go", "vsphere_volume_node_poweroff.go", "vsphere_volume_ops_storm.go", "vsphere_volume_perf.go", @@ -35,10 +42,18 @@ go_library( "//pkg/volume/util/volumehelper:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/storage/utils:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/vmware/govmomi:go_default_library", + "//vendor/github.com/vmware/govmomi/find:go_default_library", + "//vendor/github.com/vmware/govmomi/object:go_default_library", + "//vendor/github.com/vmware/govmomi/session:go_default_library", + "//vendor/github.com/vmware/govmomi/vim25:go_default_library", + "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", @@ -62,9 +77,6 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//test/e2e/storage/vsphere/bootstrap:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], ) diff --git a/test/e2e/storage/vsphere/bootstrap/bootstrap.go b/test/e2e/storage/vsphere/bootstrap.go similarity index 58% rename from test/e2e/storage/vsphere/bootstrap/bootstrap.go rename to test/e2e/storage/vsphere/bootstrap.go index c1129839e80..8d1d93a7260 100644 --- a/test/e2e/storage/vsphere/bootstrap/bootstrap.go +++ b/test/e2e/storage/vsphere/bootstrap.go @@ -14,18 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package bootstrap +package vsphere import ( + "k8s.io/kubernetes/test/e2e/framework" "sync" ) var once sync.Once var waiting = make(chan bool) +var f *framework.Framework // Bootstrap takes care of initializing necessary test context for vSphere tests -func Bootstrap() { +func Bootstrap(fw *framework.Framework) { done := make(chan bool) + f = fw go func() { once.Do(bootstrapOnce) <-waiting @@ -35,10 +38,19 @@ func Bootstrap() { } func bootstrapOnce() { - // TBD // 1. Read vSphere conf and get VSphere instances - // 2. Get Node to VSphere mapping - // 3. Set NodeMapper in vSphere context - TestContext = Context{} + vsphereInstances, err := GetVSphereInstances() + if err != nil { + framework.Failf("Failed to bootstrap vSphere with error: %v", err) + } + // 2. Get all ready nodes + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + TestContext = VSphereContext{NodeMapper: &NodeMapper{}, VSphereInstances: vsphereInstances} + + // 3. Get Node to VSphere mapping + err = TestContext.NodeMapper.GenerateNodeMap(vsphereInstances, *nodeList) + if err != nil { + framework.Failf("Failed to bootstrap vSphere with error: %v", err) + } close(waiting) } diff --git a/test/e2e/storage/vsphere/bootstrap/BUILD b/test/e2e/storage/vsphere/bootstrap/BUILD deleted file mode 100644 index 03beab680ab..00000000000 --- a/test/e2e/storage/vsphere/bootstrap/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "bootstrap.go", - "context.go", - ], - importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere/bootstrap", - visibility = ["//visibility:public"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/test/e2e/storage/vsphere/config.go b/test/e2e/storage/vsphere/config.go new file mode 100644 index 00000000000..3ee965dbd37 --- /dev/null +++ b/test/e2e/storage/vsphere/config.go @@ -0,0 +1,178 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "errors" + "fmt" + "gopkg.in/gcfg.v1" + "io" + "k8s.io/kubernetes/test/e2e/framework" + "os" +) + +const ( + vSphereConfFileEnvVar = "VSPHERE_CONF_FILE" +) + +var ( + confFileLocation = os.Getenv(vSphereConfFileEnvVar) +) + +// Config represents vSphere configuration +type Config struct { + Username string + Password string + Hostname string + Port string + Datacenters string + RoundTripperCount uint + DefaultDatastore string +} + +// ConfigFile represents the content of vsphere.conf file. +// Users specify the configuration of one or more vSphere instances in vsphere.conf where +// the Kubernetes master and worker nodes are running. +type ConfigFile struct { + Global struct { + // vCenter username. + User string `gcfg:"user"` + // vCenter password in clear text. + Password string `gcfg:"password"` + // vCenter port. + VCenterPort string `gcfg:"port"` + // True if vCenter uses self-signed cert. + InsecureFlag bool `gcfg:"insecure-flag"` + // Datacenter in which VMs are located. + Datacenters string `gcfg:"datacenters"` + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint `gcfg:"soap-roundtrip-count"` + } + + VirtualCenter map[string]*Config + + Network struct { + // PublicNetwork is name of the network the VMs are joined to. + PublicNetwork string `gcfg:"public-network"` + } + + Disk struct { + // SCSIControllerType defines SCSI controller to be used. + SCSIControllerType string `dcfg:"scsicontrollertype"` + } + + // Endpoint used to create volumes + Workspace struct { + VCenterIP string `gcfg:"server"` + Datacenter string `gcfg:"datacenter"` + Folder string `gcfg:"folder"` + DefaultDatastore string `gcfg:"default-datastore"` + ResourcePoolPath string `gcfg:"resourcepool-path"` + } +} + +// GetVSphereInstances parses vsphere.conf and returns VSphere instances +func GetVSphereInstances() (map[string]*VSphere, error) { + cfg, err := getConfig() + if err != nil { + return nil, err + } + return populateInstanceMap(cfg) +} + +func getConfig() (*ConfigFile, error) { + if confFileLocation == "" { + return nil, fmt.Errorf("Env variable 'VSPHERE_CONF_FILE' is not set.") + } + confFile, err := os.Open(confFileLocation) + if err != nil { + return nil, err + } + defer confFile.Close() + cfg, err := readConfig(confFile) + if err != nil { + return nil, err + } + return &cfg, nil +} + +// readConfig parses vSphere cloud config file into ConfigFile. +func readConfig(config io.Reader) (ConfigFile, error) { + if config == nil { + err := fmt.Errorf("no vSphere cloud provider config file given") + return ConfigFile{}, err + } + + var cfg ConfigFile + err := gcfg.ReadInto(&cfg, config) + return cfg, err +} + +func populateInstanceMap(cfg *ConfigFile) (map[string]*VSphere, error) { + vsphereInstances := make(map[string]*VSphere) + + if cfg.Workspace.VCenterIP == "" || cfg.Workspace.DefaultDatastore == "" || cfg.Workspace.Folder == "" || cfg.Workspace.Datacenter == "" { + msg := fmt.Sprintf("All fields in workspace are mandatory."+ + " vsphere.conf does not have the workspace specified correctly. cfg.Workspace: %+v", cfg.Workspace) + framework.Logf(msg) + return nil, errors.New(msg) + } + for vcServer, vcConfig := range cfg.VirtualCenter { + framework.Logf("Initializing vc server %s", vcServer) + if vcServer == "" { + framework.Logf("vsphere.conf does not have the VirtualCenter IP address specified") + return nil, errors.New("vsphere.conf does not have the VirtualCenter IP address specified") + } + vcConfig.Hostname = vcServer + + if vcConfig.Username == "" { + vcConfig.Username = cfg.Global.User + } + if vcConfig.Password == "" { + vcConfig.Password = cfg.Global.Password + } + if vcConfig.Username == "" { + msg := fmt.Sprintf("vcConfig.User is empty for vc %s!", vcServer) + framework.Logf(msg) + return nil, errors.New(msg) + } + if vcConfig.Password == "" { + msg := fmt.Sprintf("vcConfig.Password is empty for vc %s!", vcServer) + framework.Logf(msg) + return nil, errors.New(msg) + } + if vcConfig.Port == "" { + vcConfig.Port = cfg.Global.VCenterPort + } + if vcConfig.Datacenters == "" && cfg.Global.Datacenters != "" { + vcConfig.Datacenters = cfg.Global.Datacenters + } + if vcConfig.RoundTripperCount == 0 { + vcConfig.RoundTripperCount = cfg.Global.RoundTripperCount + } + + vcConfig.DefaultDatastore = cfg.Workspace.DefaultDatastore + + vsphereIns := VSphere{ + Config: vcConfig, + } + vsphereInstances[vcServer] = &vsphereIns + } + + framework.Logf("ConfigFile %v \n vSphere instances %v", cfg, vsphereInstances) + return vsphereInstances, nil +} diff --git a/test/e2e/storage/vsphere/connection.go b/test/e2e/storage/vsphere/connection.go new file mode 100644 index 00000000000..d123a7f3421 --- /dev/null +++ b/test/e2e/storage/vsphere/connection.go @@ -0,0 +1,91 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "fmt" + neturl "net/url" + "sync" + + "github.com/golang/glog" + "github.com/vmware/govmomi" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/vim25" + "golang.org/x/net/context" +) + +const ( + roundTripperDefaultCount = 3 +) + +var ( + clientLock sync.Mutex +) + +// Connect makes connection to vSphere +// No actions are taken if a connection exists and alive. Otherwise, a new client will be created. +func Connect(ctx context.Context, vs *VSphere) error { + var err error + clientLock.Lock() + defer clientLock.Unlock() + + if vs.Client == nil { + vs.Client, err = NewClient(ctx, vs) + if err != nil { + glog.Errorf("Failed to create govmomi client. err: %+v", err) + return err + } + return nil + } + manager := session.NewManager(vs.Client.Client) + userSession, err := manager.UserSession(ctx) + if err != nil { + glog.Errorf("Error while obtaining user session. err: %+v", err) + return err + } + if userSession != nil { + return nil + } + glog.Warningf("Creating new client session since the existing session is not valid or not authenticated") + vs.Client.Logout(ctx) + vs.Client, err = NewClient(ctx, vs) + if err != nil { + glog.Errorf("Failed to create govmomi client. err: %+v", err) + return err + } + return nil +} + +// NewClient creates a new client for vSphere connection +func NewClient(ctx context.Context, vs *VSphere) (*govmomi.Client, error) { + url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", vs.Config.Hostname, vs.Config.Port)) + if err != nil { + glog.Errorf("Failed to parse URL: %s. err: %+v", url, err) + return nil, err + } + url.User = neturl.UserPassword(vs.Config.Username, vs.Config.Password) + client, err := govmomi.NewClient(ctx, url, true) + if err != nil { + glog.Errorf("Failed to create new client. err: %+v", err) + return nil, err + } + if vs.Config.RoundTripperCount == 0 { + vs.Config.RoundTripperCount = roundTripperDefaultCount + } + client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(int(vs.Config.RoundTripperCount))) + return client, nil +} diff --git a/test/e2e/storage/vsphere/bootstrap/context.go b/test/e2e/storage/vsphere/context.go similarity index 84% rename from test/e2e/storage/vsphere/bootstrap/context.go rename to test/e2e/storage/vsphere/context.go index fca4c895ce5..c6ee06f21da 100644 --- a/test/e2e/storage/vsphere/bootstrap/context.go +++ b/test/e2e/storage/vsphere/context.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package bootstrap +package vsphere // Context holds common information for vSphere tests -type Context struct { - // NodeMapper and other instances, common to vSphere tests +type VSphereContext struct { + NodeMapper *NodeMapper + VSphereInstances map[string]*VSphere } // TestContext should be used by all tests to access common context data. It should be initialized only once, during bootstrapping the tests. -var TestContext Context +var TestContext VSphereContext diff --git a/test/e2e/storage/vsphere/nodemapper.go b/test/e2e/storage/vsphere/nodemapper.go new file mode 100644 index 00000000000..7c197fd2d54 --- /dev/null +++ b/test/e2e/storage/vsphere/nodemapper.go @@ -0,0 +1,134 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + "golang.org/x/net/context" + "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" + "strings" + "sync" +) + +type NodeMapper struct { +} + +type NodeInfo struct { + Name string + DataCenterRef types.ManagedObjectReference + VirtualMachineRef types.ManagedObjectReference + VSphere *VSphere +} + +var ( + nameToNodeInfo = make(map[string]*NodeInfo) +) + +// GenerateNodeMap populates node name to node info map +func (nm *NodeMapper) GenerateNodeMap(vSphereInstances map[string]*VSphere, nodeList v1.NodeList) error { + type VmSearch struct { + vs *VSphere + datacenter *object.Datacenter + } + + var wg sync.WaitGroup + var queueChannel []*VmSearch + + var datacenters []*object.Datacenter + var err error + for _, vs := range vSphereInstances { + + // Create context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if vs.Config.Datacenters == "" { + datacenters, err = vs.GetAllDatacenter(ctx) + if err != nil { + framework.Logf("NodeMapper error: %v", err) + continue + } + } else { + dcName := strings.Split(vs.Config.Datacenters, ",") + for _, dc := range dcName { + dc = strings.TrimSpace(dc) + if dc == "" { + continue + } + datacenter, err := vs.GetDatacenter(ctx, dc) + if err != nil { + framework.Logf("NodeMapper error dc: %s \n err: %v", dc, err) + + continue + } + datacenters = append(datacenters, datacenter) + } + } + + for _, dc := range datacenters { + framework.Logf("Search candidates vc=%s and datacenter=%s", vs.Config.Hostname, dc.Name()) + queueChannel = append(queueChannel, &VmSearch{vs: vs, datacenter: dc}) + } + } + + for _, node := range nodeList.Items { + n := node + + go func() { + nodeUUID := n.Status.NodeInfo.SystemUUID + framework.Logf("Searching for node with UUID: %s", nodeUUID) + for _, res := range queueChannel { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + vm, err := res.vs.GetVMByUUID(ctx, nodeUUID, res.datacenter) + if err != nil { + framework.Logf("Error %v while looking for node=%s in vc=%s and datacenter=%s", + err, n.Name, res.vs.Config.Hostname, res.datacenter.Name()) + continue + } + if vm != nil { + framework.Logf("Found node %s as vm=%+v in vc=%s and datacenter=%s", + n.Name, vm, res.vs.Config.Hostname, res.datacenter.Name()) + nodeInfo := &NodeInfo{Name: n.Name, DataCenterRef: res.datacenter.Reference(), VirtualMachineRef: vm.Reference(), VSphere: res.vs} + nameToNodeInfo[n.Name] = nodeInfo + break + } + } + wg.Done() + }() + wg.Add(1) + } + wg.Wait() + + if len(nameToNodeInfo) != len(nodeList.Items) { + return errors.New("all nodes not mapped to respective vSphere") + } + return nil +} + +// GetNodeInfo return NodeInfo for given nodeName +func (nm *NodeMapper) GetNodeInfo(nodeName string) *NodeInfo { + return nameToNodeInfo[nodeName] +} + +// SetNodeInfo sets NodeInfo for given nodeName. This function is not thread safe. Users need to handle concurrency. +func (nm *NodeMapper) SetNodeInfo(nodeName string, nodeInfo *NodeInfo) { + nameToNodeInfo[nodeName] = nodeInfo +} diff --git a/test/e2e/storage/vsphere/vsphere.go b/test/e2e/storage/vsphere/vsphere.go new file mode 100644 index 00000000000..eb50c1355b3 --- /dev/null +++ b/test/e2e/storage/vsphere/vsphere.go @@ -0,0 +1,54 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "github.com/vmware/govmomi" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "golang.org/x/net/context" + "strings" +) + +// Represents a vSphere instance where one or more kubernetes nodes are running. +type VSphere struct { + Config *Config + Client *govmomi.Client +} + +// GetDatacenter returns the DataCenter Object for the given datacenterPath +func (vs *VSphere) GetDatacenter(ctx context.Context, datacenterPath string) (*object.Datacenter, error) { + Connect(ctx, vs) + finder := find.NewFinder(vs.Client.Client, true) + return finder.Datacenter(ctx, datacenterPath) +} + +// GetAllDatacenter returns all the DataCenter Objects +func (vs *VSphere) GetAllDatacenter(ctx context.Context) ([]*object.Datacenter, error) { + Connect(ctx, vs) + finder := find.NewFinder(vs.Client.Client, true) + return finder.DatacenterList(ctx, "*") +} + +// GetVMByUUID gets the VM object from the given vmUUID +func (vs *VSphere) GetVMByUUID(ctx context.Context, vmUUID string, dc object.Reference) (object.Reference, error) { + Connect(ctx, vs) + datacenter := object.NewDatacenter(vs.Client.Client, dc.Reference()) + s := object.NewSearchIndex(vs.Client.Client) + vmUUID = strings.ToLower(strings.TrimSpace(vmUUID)) + return s.FindByUuid(ctx, datacenter, vmUUID, true, nil) +} diff --git a/test/e2e/storage/vsphere/vsphere_utils.go b/test/e2e/storage/vsphere/vsphere_utils.go index 57dc6ddef0a..6dd95b4e33a 100644 --- a/test/e2e/storage/vsphere/vsphere_utils.go +++ b/test/e2e/storage/vsphere/vsphere_utils.go @@ -23,6 +23,8 @@ import ( "time" . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -36,6 +38,11 @@ import ( "k8s.io/kubernetes/pkg/volume/util/volumehelper" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" + + "context" + + "github.com/vmware/govmomi/find" + vimtypes "github.com/vmware/govmomi/vim25/types" ) const ( @@ -467,3 +474,96 @@ func getVSphere(c clientset.Interface) (*vsphere.VSphere, error) { func GetVSphere(c clientset.Interface) (*vsphere.VSphere, error) { return getVSphere(c) } + +// get .vmx file path for a virtual machine +func getVMXFilePath(vmObject *object.VirtualMachine) (vmxPath string) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var nodeVM mo.VirtualMachine + err := vmObject.Properties(ctx, vmObject.Reference(), []string{"config.files"}, &nodeVM) + Expect(err).NotTo(HaveOccurred()) + Expect(nodeVM.Config).NotTo(BeNil()) + + vmxPath = nodeVM.Config.Files.VmPathName + framework.Logf("vmx file path is %s", vmxPath) + return vmxPath +} + +// verify ready node count. Try upto 3 minutes. Return true if count is expected count +func verifyReadyNodeCount(client clientset.Interface, expectedNodes int) bool { + numNodes := 0 + for i := 0; i < 36; i++ { + nodeList := framework.GetReadySchedulableNodesOrDie(client) + Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node") + + numNodes = len(nodeList.Items) + if numNodes == expectedNodes { + break + } + time.Sleep(5 * time.Second) + } + return (numNodes == expectedNodes) +} + +// poweroff nodeVM and confirm the poweroff state +func poweroffNodeVM(nodeName string, vm *object.VirtualMachine) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Powering off node VM %s", nodeName) + + _, err := vm.PowerOff(ctx) + Expect(err).NotTo(HaveOccurred()) + err = vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOff) + Expect(err).NotTo(HaveOccurred(), "Unable to power off the node") +} + +// poweron nodeVM and confirm the poweron state +func poweronNodeVM(nodeName string, vm *object.VirtualMachine) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Powering on node VM %s", nodeName) + + vm.PowerOn(ctx) + err := vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOn) + Expect(err).NotTo(HaveOccurred(), "Unable to power on the node") +} + +// unregister a nodeVM from VC +func unregisterNodeVM(nodeName string, vm *object.VirtualMachine) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + poweroffNodeVM(nodeName, vm) + + framework.Logf("Unregistering node VM %s", nodeName) + err := vm.Unregister(ctx) + Expect(err).NotTo(HaveOccurred(), "Unable to unregister the node") +} + +// register a nodeVM into a VC +func registerNodeVM(nodeName, workingDir, vmxFilePath string, rpool *object.ResourcePool, host *object.HostSystem) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Registering node VM %s with vmx file path %s", nodeName, vmxFilePath) + + nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeName) + finder := find.NewFinder(nodeInfo.VSphere.Client.Client, true) + + vmFolder, err := finder.FolderOrDefault(ctx, workingDir) + Expect(err).NotTo(HaveOccurred()) + + registerTask, err := vmFolder.RegisterVM(ctx, vmxFilePath, nodeName, false, rpool, host) + Expect(err).NotTo(HaveOccurred()) + err = registerTask.Wait(ctx) + Expect(err).NotTo(HaveOccurred()) + + vmPath := filepath.Join(workingDir, nodeName) + vm, err := finder.VirtualMachine(ctx, vmPath) + Expect(err).NotTo(HaveOccurred()) + + poweronNodeVM(nodeName, vm) +} diff --git a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go index 002a178023c..7f2bed4d869 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go +++ b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go @@ -51,6 +51,7 @@ var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:v BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name scParameters = make(map[string]string) diff --git a/test/e2e/storage/vsphere/vsphere_volume_datastore.go b/test/e2e/storage/vsphere/vsphere_volume_datastore.go index 70f55ee9f8d..8ac75de2983 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_datastore.go +++ b/test/e2e/storage/vsphere/vsphere_volume_datastore.go @@ -54,6 +54,7 @@ var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name scParameters = make(map[string]string) diff --git a/test/e2e/storage/vsphere/vsphere_volume_diskformat.go b/test/e2e/storage/vsphere/vsphere_volume_diskformat.go index a1dd4ea94f4..fb5a611c09e 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_diskformat.go +++ b/test/e2e/storage/vsphere/vsphere_volume_diskformat.go @@ -65,6 +65,7 @@ var _ = utils.SIGDescribe("Volume Disk Format [Feature:vsphere]", func() { ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) diff --git a/test/e2e/storage/vsphere/vsphere_volume_disksize.go b/test/e2e/storage/vsphere/vsphere_volume_disksize.go index 14da7f82e18..107335d5bc0 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_disksize.go +++ b/test/e2e/storage/vsphere/vsphere_volume_disksize.go @@ -54,6 +54,7 @@ var _ = utils.SIGDescribe("Volume Disk Size [Feature:vsphere]", func() { ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name scParameters = make(map[string]string) diff --git a/test/e2e/storage/vsphere/vsphere_volume_fstype.go b/test/e2e/storage/vsphere/vsphere_volume_fstype.go index 6ec152983a8..af98ea62635 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_fstype.go +++ b/test/e2e/storage/vsphere/vsphere_volume_fstype.go @@ -72,6 +72,7 @@ var _ = utils.SIGDescribe("Volume FStype [Feature:vsphere]", func() { ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) diff --git a/test/e2e/storage/vsphere/vsphere_volume_master_restart.go b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go index cb12e9d0edd..48ab67f8a7a 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_master_restart.go +++ b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go @@ -57,6 +57,7 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout)) diff --git a/test/e2e/storage/vsphere/vsphere_volume_node_delete.go b/test/e2e/storage/vsphere/vsphere_volume_node_delete.go new file mode 100644 index 00000000000..fe166e8d980 --- /dev/null +++ b/test/e2e/storage/vsphere/vsphere_volume_node_delete.go @@ -0,0 +1,119 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "golang.org/x/net/context" + + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +var _ = utils.SIGDescribe("Node Unregister [Feature:vsphere] [Slow] [Disruptive]", func() { + f := framework.NewDefaultFramework("node-unregister") + var ( + client clientset.Interface + namespace string + workingDir string + err error + ) + + BeforeEach(func() { + framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) + client = f.ClientSet + namespace = f.Namespace.Name + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout)) + Expect(err).NotTo(HaveOccurred()) + workingDir = os.Getenv("VSPHERE_WORKING_DIR") + Expect(workingDir).NotTo(BeEmpty()) + + }) + + It("node unregister", func() { + By("Get total Ready nodes") + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + Expect(len(nodeList.Items) > 1).To(BeTrue(), "At least 2 nodes are required for this test") + + totalNodesCount := len(nodeList.Items) + nodeVM := nodeList.Items[0] + + nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeVM.ObjectMeta.Name) + vmObject := object.NewVirtualMachine(nodeInfo.VSphere.Client.Client, nodeInfo.VirtualMachineRef) + + // Find VM .vmx file path, host, resource pool. + // They are required to register a node VM to VC + vmxFilePath := getVMXFilePath(vmObject) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + vmHost, err := vmObject.HostSystem(ctx) + Expect(err).NotTo(HaveOccurred()) + + vmPool, err := vmObject.ResourcePool(ctx) + Expect(err).NotTo(HaveOccurred()) + + // Unregister Node VM + By("Unregister a node VM") + unregisterNodeVM(nodeVM.ObjectMeta.Name, vmObject) + + // Ready nodes should be 1 less + By("Verifying the ready node counts") + Expect(verifyReadyNodeCount(f.ClientSet, totalNodesCount-1)).To(BeTrue(), "Unable to verify expected ready node count") + + nodeList = framework.GetReadySchedulableNodesOrDie(client) + Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node") + + var nodeNameList []string + for _, node := range nodeList.Items { + nodeNameList = append(nodeNameList, node.ObjectMeta.Name) + } + Expect(nodeNameList).NotTo(ContainElement(nodeVM.ObjectMeta.Name)) + + // Register Node VM + By("Register back the node VM") + registerNodeVM(nodeVM.ObjectMeta.Name, workingDir, vmxFilePath, vmPool, vmHost) + + // Ready nodes should be equal to earlier count + By("Verifying the ready node counts") + Expect(verifyReadyNodeCount(f.ClientSet, totalNodesCount)).To(BeTrue(), "Unable to verify expected ready node count") + + nodeList = framework.GetReadySchedulableNodesOrDie(client) + Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node") + + nodeNameList = nodeNameList[:0] + for _, node := range nodeList.Items { + nodeNameList = append(nodeNameList, node.ObjectMeta.Name) + } + Expect(nodeNameList).To(ContainElement(nodeVM.ObjectMeta.Name)) + + // Sanity test that pod provisioning works + By("Sanity check for volume lifecycle") + scParameters := make(map[string]string) + storagePolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY") + Expect(storagePolicy).NotTo(BeEmpty(), "Please set VSPHERE_SPBM_GOLD_POLICY system environment") + scParameters[SpbmStoragePolicy] = storagePolicy + invokeValidPolicyTest(f, client, namespace, scParameters) + }) +}) diff --git a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go index 42140257fbd..d767e9f430d 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go +++ b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go @@ -52,6 +52,7 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout)) diff --git a/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go b/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go index 80a07ceeec0..4ff76399496 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go +++ b/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go @@ -63,6 +63,7 @@ var _ = utils.SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() { ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) diff --git a/test/e2e/storage/vsphere/vsphere_volume_perf.go b/test/e2e/storage/vsphere/vsphere_volume_perf.go index ed466b64c39..105f7b14ac5 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_perf.go +++ b/test/e2e/storage/vsphere/vsphere_volume_perf.go @@ -63,6 +63,7 @@ var _ = utils.SIGDescribe("vcp-performance [Feature:vsphere]", func() { BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name diff --git a/test/e2e/storage/vsphere/vsphere_volume_placement.go b/test/e2e/storage/vsphere/vsphere_volume_placement.go index 8a2b3f245a5..d2afa3171d5 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_placement.go +++ b/test/e2e/storage/vsphere/vsphere_volume_placement.go @@ -52,6 +52,7 @@ var _ = utils.SIGDescribe("Volume Placement", func() { ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) c = f.ClientSet ns = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) diff --git a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go index 3c51cbce4eb..adc961fde80 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go +++ b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go @@ -98,6 +98,7 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name policyName = GetAndExpectStringEnvVar(SPBMPolicyName)