From 4683a9471f40cb716331e4b6d44f70fd9d75ea42 Mon Sep 17 00:00:00 2001 From: Balu Dontu Date: Thu, 18 Jan 2018 15:29:27 -0800 Subject: [PATCH] Add multi-vc configuration for e2e tests --- hack/verify-flags/excluded-flags.txt | 1 + pkg/cloudprovider/providers/vsphere/BUILD | 1 - .../providers/vsphere/nodemanager.go | 7 ++ .../providers/vsphere/vsphere.go | 7 ++ .../providers/vsphere/vsphere_util.go | 114 +++++++++--------- test/e2e/storage/vsphere/BUILD | 2 +- test/e2e/storage/vsphere/vsphere_common.go | 66 ++++++++++ test/e2e/storage/vsphere/vsphere_scale.go | 28 ++--- test/e2e/storage/vsphere/vsphere_stress.go | 21 +--- .../vsphere/vsphere_volume_cluster_ds.go | 21 ++-- .../vsphere/vsphere_volume_diskformat.go | 21 ++-- .../vsphere/vsphere_volume_disksize.go | 4 +- .../vsphere/vsphere_volume_node_poweroff.go | 24 +--- .../storage/vsphere/vsphere_volume_perf.go | 30 ++--- .../vsphere/vsphere_volume_placement.go | 3 +- .../vsphere/vsphere_volume_vsan_policy.go | 96 ++++++--------- 16 files changed, 221 insertions(+), 225 deletions(-) create mode 100644 test/e2e/storage/vsphere/vsphere_common.go diff --git a/hack/verify-flags/excluded-flags.txt b/hack/verify-flags/excluded-flags.txt index 7047134c541..687e24ff7be 100644 --- a/hack/verify-flags/excluded-flags.txt +++ b/hack/verify-flags/excluded-flags.txt @@ -23,3 +23,4 @@ retry_time file_content_in_loop break_on_expected_content Premium_LRS +VCP_STRESS_ITERATIONS diff --git a/pkg/cloudprovider/providers/vsphere/BUILD b/pkg/cloudprovider/providers/vsphere/BUILD index 2ccae8c2f2d..b24ec93eb16 100644 --- a/pkg/cloudprovider/providers/vsphere/BUILD +++ b/pkg/cloudprovider/providers/vsphere/BUILD @@ -21,7 +21,6 @@ go_library( "//pkg/cloudprovider/providers/vsphere/vclib/diskmanagers:go_default_library", "//pkg/controller:go_default_library", "//vendor/github.com/golang/glog:go_default_library", - "//vendor/github.com/vmware/govmomi:go_default_library", "//vendor/github.com/vmware/govmomi/vim25:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index a30b9135b01..d2c1770bc20 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -354,3 +354,10 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI vm := nodeInfo.vm.RenewVM(vsphereInstance.conn.GoVmomiClient) return &NodeInfo{vm: &vm, dataCenter: vm.Datacenter, vcServer: nodeInfo.vcServer}, nil } + +func (nodeInfo *NodeInfo) VM() *vclib.VirtualMachine { + if nodeInfo == nil { + return nil + } + return nodeInfo.vm +} diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index 42400682cb7..ad48e0bae72 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -1159,3 +1159,10 @@ func (vs *VSphere) NodeDeleted(obj interface{}) { glog.V(4).Infof("Node deleted: %+v", node) vs.nodeManager.UnRegisterNode(node) } + +func (vs *VSphere) NodeManager() (nodeManager *NodeManager) { + if vs == nil { + return nil + } + return vs.nodeManager +} diff --git a/pkg/cloudprovider/providers/vsphere/vsphere_util.go b/pkg/cloudprovider/providers/vsphere/vsphere_util.go index 45a35c71d41..069411523a3 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere_util.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere_util.go @@ -21,12 +21,10 @@ import ( "errors" "os" "regexp" - "runtime" "strings" "time" "github.com/golang/glog" - "github.com/vmware/govmomi" "github.com/vmware/govmomi/vim25" "fmt" @@ -34,7 +32,6 @@ import ( "path/filepath" "github.com/vmware/govmomi/vim25/mo" - "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers" @@ -46,63 +43,37 @@ const ( Folder = "Folder" VirtualMachine = "VirtualMachine" DummyDiskName = "kube-dummyDisk.vmdk" + vSphereConfFileEnvVar = "VSPHERE_CONF_FILE" ) // GetVSphere reads vSphere configuration from system environment and construct vSphere object func GetVSphere() (*VSphere, error) { - cfg := getVSphereConfig() - vSphereConn := getVSphereConn(cfg) - client, err := GetgovmomiClient(vSphereConn) + cfg, err := getVSphereConfig() if err != nil { return nil, err } - vSphereConn.GoVmomiClient = client - vsphereIns := &VSphereInstance{ - conn: vSphereConn, - cfg: &VirtualCenterConfig{ - User: cfg.Global.User, - Password: cfg.Global.Password, - VCenterPort: cfg.Global.VCenterPort, - Datacenters: cfg.Global.Datacenters, - RoundTripperCount: cfg.Global.RoundTripperCount, - }, + vs, err := newControllerNode(*cfg) + if err != nil { + return nil, err } - vsphereInsMap := make(map[string]*VSphereInstance) - vsphereInsMap[cfg.Global.VCenterIP] = vsphereIns - // TODO: Initialize nodeManager and set it in VSphere. - vs := &VSphere{ - vsphereInstanceMap: vsphereInsMap, - hostName: "", - cfg: cfg, - nodeManager: &NodeManager{ - vsphereInstanceMap: vsphereInsMap, - nodeInfoMap: make(map[string]*NodeInfo), - registeredNodes: make(map[string]*v1.Node), - }, - } - runtime.SetFinalizer(vs, logout) return vs, nil } -func getVSphereConfig() *VSphereConfig { - var cfg VSphereConfig - cfg.Global.VCenterIP = os.Getenv("VSPHERE_VCENTER") - cfg.Global.VCenterPort = os.Getenv("VSPHERE_VCENTER_PORT") - cfg.Global.User = os.Getenv("VSPHERE_USER") - cfg.Global.Password = os.Getenv("VSPHERE_PASSWORD") - cfg.Global.Datacenters = os.Getenv("VSPHERE_DATACENTER") - cfg.Global.DefaultDatastore = os.Getenv("VSPHERE_DATASTORE") - cfg.Global.WorkingDir = os.Getenv("VSPHERE_WORKING_DIR") - cfg.Global.VMName = os.Getenv("VSPHERE_VM_NAME") - cfg.Global.InsecureFlag = false - if strings.ToLower(os.Getenv("VSPHERE_INSECURE")) == "true" { - cfg.Global.InsecureFlag = true +func getVSphereConfig() (*VSphereConfig, error) { + confFileLocation := os.Getenv(vSphereConfFileEnvVar) + if confFileLocation == "" { + return nil, fmt.Errorf("Env variable 'VSPHERE_CONF_FILE' is not set.") } - cfg.Workspace.VCenterIP = cfg.Global.VCenterIP - cfg.Workspace.Datacenter = cfg.Global.Datacenters - cfg.Workspace.DefaultDatastore = cfg.Global.DefaultDatastore - cfg.Workspace.Folder = cfg.Global.WorkingDir - return &cfg + confFile, err := os.Open(confFileLocation) + if err != nil { + return nil, err + } + defer confFile.Close() + cfg, err := readConfig(confFile) + if err != nil { + return nil, err + } + return &cfg, nil } func getVSphereConn(cfg *VSphereConfig) *vclib.VSphereConnection { @@ -117,16 +88,6 @@ func getVSphereConn(cfg *VSphereConfig) *vclib.VSphereConnection { return vSphereConn } -// GetgovmomiClient gets the goVMOMI client for the vsphere connection object -func GetgovmomiClient(conn *vclib.VSphereConnection) (*govmomi.Client, error) { - if conn == nil { - cfg := getVSphereConfig() - conn = getVSphereConn(cfg) - } - client, err := conn.NewClient(context.TODO()) - return client, err -} - // Returns the accessible datastores for the given node VM. func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { accessibleDatastores, err := nodeVmDetail.vm.GetAllAccessibleDatastores(ctx) @@ -512,3 +473,40 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN } return nodesToRetry, nil } + +func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) { + isDummyVMPresent := false + + // Create context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) + if err != nil { + return isDummyVMPresent, err + } + + dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) + if err != nil { + return isDummyVMPresent, err + } + + vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) + if err != nil { + return isDummyVMPresent, err + } + + vms, err := vmFolder.GetVirtualMachines(ctx) + if err != nil { + return isDummyVMPresent, err + } + + for _, vm := range vms { + if vm.Name() == vmName { + isDummyVMPresent = true + break + } + } + + return isDummyVMPresent, nil +} diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD index d582d820046..66c6bdf4f9a 100644 --- a/test/e2e/storage/vsphere/BUILD +++ b/test/e2e/storage/vsphere/BUILD @@ -11,6 +11,7 @@ go_library( "persistent_volumes-vsphere.go", "pv_reclaimpolicy.go", "pvc_label_selector.go", + "vsphere_common.go", "vsphere_scale.go", "vsphere_statefulsets.go", "vsphere_stress.go", @@ -36,7 +37,6 @@ go_library( "//test/e2e/storage/utils:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", - "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/test/e2e/storage/vsphere/vsphere_common.go b/test/e2e/storage/vsphere/vsphere_common.go new file mode 100644 index 00000000000..3c93cab6cea --- /dev/null +++ b/test/e2e/storage/vsphere/vsphere_common.go @@ -0,0 +1,66 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + . "github.com/onsi/gomega" + "os" + "strconv" +) + +const ( + SPBMPolicyName = "VSPHERE_SPBM_POLICY_NAME" + StorageClassDatastoreName = "VSPHERE_DATASTORE" + SecondSharedDatastore = "VSPHERE_SECOND_SHARED_DATASTORE" + KubernetesClusterName = "VSPHERE_KUBERNETES_CLUSTER" + SPBMTagPolicy = "VSPHERE_SPBM_TAG_POLICY" +) + +const ( + VCPClusterDatastore = "CLUSTER_DATASTORE" + SPBMPolicyDataStoreCluster = "VSPHERE_SPBM_POLICY_DS_CLUSTER" +) + +const ( + VCPScaleVolumeCount = "VCP_SCALE_VOLUME_COUNT" + VCPScaleVolumesPerPod = "VCP_SCALE_VOLUME_PER_POD" + VCPScaleInstances = "VCP_SCALE_INSTANCES" +) + +const ( + VCPStressInstances = "VCP_STRESS_INSTANCES" + VCPStressIterations = "VCP_STRESS_ITERATIONS" +) + +const ( + VCPPerfVolumeCount = "VCP_PERF_VOLUME_COUNT" + VCPPerfVolumesPerPod = "VCP_PERF_VOLUME_PER_POD" + VCPPerfIterations = "VCP_PERF_ITERATIONS" +) + +func GetAndExpectStringEnvVar(varName string) string { + varValue := os.Getenv(varName) + Expect(varValue).NotTo(BeEmpty(), "ENV "+varName+" is not set") + return varValue +} + +func GetAndExpectIntEnvVar(varName string) int { + varValue := GetAndExpectStringEnvVar(varName) + varIntValue, err := strconv.Atoi(varValue) + Expect(err).NotTo(HaveOccurred(), "Error Parsing "+varName) + return varIntValue +} diff --git a/test/e2e/storage/vsphere/vsphere_scale.go b/test/e2e/storage/vsphere/vsphere_scale.go index 2da0933f87a..6d5548b8711 100644 --- a/test/e2e/storage/vsphere/vsphere_scale.go +++ b/test/e2e/storage/vsphere/vsphere_scale.go @@ -18,7 +18,6 @@ package vsphere import ( "fmt" - "os" "strconv" . "github.com/onsi/ginkgo" @@ -63,12 +62,11 @@ var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() { volumeCount int numberOfInstances int volumesPerPod int - nodeVolumeMapChan chan map[string][]string - nodes *v1.NodeList policyName string datastoreName string + nodeVolumeMapChan chan map[string][]string + nodes *v1.NodeList scNames = []string{storageclass1, storageclass2, storageclass3, storageclass4} - err error ) BeforeEach(func() { @@ -78,27 +76,15 @@ var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() { nodeVolumeMapChan = make(chan map[string][]string) // Read the environment variables - volumeCountStr := os.Getenv("VCP_SCALE_VOLUME_COUNT") - Expect(volumeCountStr).NotTo(BeEmpty(), "ENV VCP_SCALE_VOLUME_COUNT is not set") - volumeCount, err = strconv.Atoi(volumeCountStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_VOLUME_COUNT") + volumeCount = GetAndExpectIntEnvVar(VCPScaleVolumeCount) + volumesPerPod = GetAndExpectIntEnvVar(VCPScaleVolumesPerPod) - volumesPerPodStr := os.Getenv("VCP_SCALE_VOLUME_PER_POD") - Expect(volumesPerPodStr).NotTo(BeEmpty(), "ENV VCP_SCALE_VOLUME_PER_POD is not set") - volumesPerPod, err = strconv.Atoi(volumesPerPodStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_VOLUME_PER_POD") - - numberOfInstancesStr := os.Getenv("VCP_SCALE_INSTANCES") - Expect(numberOfInstancesStr).NotTo(BeEmpty(), "ENV VCP_SCALE_INSTANCES is not set") - numberOfInstances, err = strconv.Atoi(numberOfInstancesStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_INSTANCES") + numberOfInstances = GetAndExpectIntEnvVar(VCPScaleInstances) Expect(numberOfInstances > 5).NotTo(BeTrue(), "Maximum allowed instances are 5") Expect(numberOfInstances > volumeCount).NotTo(BeTrue(), "Number of instances should be less than the total volume count") - policyName = os.Getenv("VSPHERE_SPBM_POLICY_NAME") - datastoreName = os.Getenv("VSPHERE_DATASTORE") - Expect(policyName).NotTo(BeEmpty(), "ENV VSPHERE_SPBM_POLICY_NAME is not set") - Expect(datastoreName).NotTo(BeEmpty(), "ENV VSPHERE_DATASTORE is not set") + policyName = GetAndExpectStringEnvVar(SPBMPolicyName) + datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName) nodes = framework.GetReadySchedulableNodesOrDie(client) if len(nodes.Items) < 2 { diff --git a/test/e2e/storage/vsphere/vsphere_stress.go b/test/e2e/storage/vsphere/vsphere_stress.go index f37597728c8..0eb7a1581ff 100644 --- a/test/e2e/storage/vsphere/vsphere_stress.go +++ b/test/e2e/storage/vsphere/vsphere_stress.go @@ -18,8 +18,6 @@ package vsphere import ( "fmt" - "os" - "strconv" "sync" . "github.com/onsi/ginkgo" @@ -50,10 +48,10 @@ var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", fun namespace string instances int iterations int - err error - scNames = []string{storageclass1, storageclass2, storageclass3, storageclass4} policyName string datastoreName string + err error + scNames = []string{storageclass1, storageclass2, storageclass3, storageclass4} ) BeforeEach(func() { @@ -67,23 +65,16 @@ var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", fun // if VCP_STRESS_INSTANCES = 12 and VCP_STRESS_ITERATIONS is 10. 12 threads will run in parallel for 10 times. // Resulting 120 Volumes and POD Creation. Volumes will be provisioned with each different types of Storage Class, // Each iteration creates PVC, verify PV is provisioned, then creates a pod, verify volume is attached to the node, and then delete the pod and delete pvc. - instancesStr := os.Getenv("VCP_STRESS_INSTANCES") - Expect(instancesStr).NotTo(BeEmpty(), "ENV VCP_STRESS_INSTANCES is not set") - instances, err = strconv.Atoi(instancesStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP-STRESS-INSTANCES") + instances = GetAndExpectIntEnvVar(VCPStressInstances) Expect(instances <= volumesPerNode*len(nodeList.Items)).To(BeTrue(), fmt.Sprintf("Number of Instances should be less or equal: %v", volumesPerNode*len(nodeList.Items))) Expect(instances > len(scNames)).To(BeTrue(), "VCP_STRESS_INSTANCES should be greater than 3 to utilize all 4 types of storage classes") - iterationStr := os.Getenv("VCP_STRESS_ITERATIONS") - Expect(instancesStr).NotTo(BeEmpty(), "ENV VCP_STRESS_ITERATIONS is not set") - iterations, err = strconv.Atoi(iterationStr) + iterations = GetAndExpectIntEnvVar(VCPStressIterations) Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_STRESS_ITERATIONS") Expect(iterations > 0).To(BeTrue(), "VCP_STRESS_ITERATIONS should be greater than 0") - policyName = os.Getenv("VSPHERE_SPBM_POLICY_NAME") - datastoreName = os.Getenv("VSPHERE_DATASTORE") - Expect(policyName).NotTo(BeEmpty(), "ENV VSPHERE_SPBM_POLICY_NAME is not set") - Expect(datastoreName).NotTo(BeEmpty(), "ENV VSPHERE_DATASTORE is not set") + policyName = GetAndExpectStringEnvVar(SPBMPolicyName) + datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName) }) It("vsphere stress tests", func() { diff --git a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go index d200eded4e1..002a178023c 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go +++ b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go @@ -18,7 +18,6 @@ package vsphere import ( "fmt" - "os" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -43,18 +42,19 @@ import ( var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-provision") - var client clientset.Interface - var namespace string - var scParameters map[string]string - var clusterDatastore string + var ( + client clientset.Interface + namespace string + scParameters map[string]string + clusterDatastore string + ) + BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") client = f.ClientSet namespace = f.Namespace.Name scParameters = make(map[string]string) - - clusterDatastore = os.Getenv("CLUSTER_DATASTORE") - Expect(clusterDatastore).NotTo(BeEmpty(), "Please set CLUSTER_DATASTORE system environment. eg: export CLUSTER_DATASTORE=/ 0) { framework.Failf("Unable to find ready and schedulable Node") diff --git a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go index c0534a73d35..23e7bd0824c 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go +++ b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go @@ -18,13 +18,10 @@ package vsphere import ( "fmt" - "os" - "path/filepath" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vmware/govmomi/find" "golang.org/x/net/context" vimtypes "github.com/vmware/govmomi/vim25/types" @@ -47,11 +44,10 @@ import ( var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", func() { f := framework.NewDefaultFramework("node-poweroff") var ( - client clientset.Interface - namespace string - vsp *vsphere.VSphere - workingDir string - err error + client clientset.Interface + namespace string + vsp *vsphere.VSphere + err error ) BeforeEach(func() { @@ -64,8 +60,6 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", Expect(len(nodeList.Items) > 1).To(BeTrue(), "At least 2 nodes are required for this test") vsp, err = getVSphere(client) Expect(err).NotTo(HaveOccurred()) - workingDir = os.Getenv("VSPHERE_WORKING_DIR") - Expect(workingDir).NotTo(BeEmpty()) }) /* @@ -118,16 +112,10 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", Expect(isAttached).To(BeTrue(), "Disk is not attached to the node") By(fmt.Sprintf("Power off the node: %v", node1)) - govMoMiClient, err := vsphere.GetgovmomiClient(nil) + nodeInfo, err := vsp.NodeManager().GetNodeInfo(node1) Expect(err).NotTo(HaveOccurred()) - - f := find.NewFinder(govMoMiClient.Client, true) + vm := nodeInfo.VM() ctx, _ := context.WithCancel(context.Background()) - - vmPath := filepath.Join(workingDir, string(node1)) - vm, err := f.VirtualMachine(ctx, vmPath) - Expect(err).NotTo(HaveOccurred()) - _, err = vm.PowerOff(ctx) Expect(err).NotTo(HaveOccurred()) defer vm.PowerOn(ctx) diff --git a/test/e2e/storage/vsphere/vsphere_volume_perf.go b/test/e2e/storage/vsphere/vsphere_volume_perf.go index 73c8da97c0b..ed466b64c39 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_perf.go +++ b/test/e2e/storage/vsphere/vsphere_volume_perf.go @@ -18,8 +18,6 @@ package vsphere import ( "fmt" - "os" - "strconv" "time" . "github.com/onsi/ginkgo" @@ -56,39 +54,25 @@ var _ = utils.SIGDescribe("vcp-performance [Feature:vsphere]", func() { client clientset.Interface namespace string nodeSelectorList []*NodeSelector + policyName string + datastoreName string volumeCount int volumesPerPod int iterations int - policyName string - datastoreName string ) BeforeEach(func() { - var err error framework.SkipUnlessProviderIs("vsphere") client = f.ClientSet namespace = f.Namespace.Name // Read the environment variables - volumeCountStr := os.Getenv("VCP_PERF_VOLUME_COUNT") - Expect(volumeCountStr).NotTo(BeEmpty(), "ENV VCP_PERF_VOLUME_COUNT is not set") - volumeCount, err = strconv.Atoi(volumeCountStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_PERF_VOLUME_COUNT") + volumeCount = GetAndExpectIntEnvVar(VCPPerfVolumeCount) + volumesPerPod = GetAndExpectIntEnvVar(VCPPerfVolumesPerPod) + iterations = GetAndExpectIntEnvVar(VCPPerfIterations) - volumesPerPodStr := os.Getenv("VCP_PERF_VOLUME_PER_POD") - Expect(volumesPerPodStr).NotTo(BeEmpty(), "ENV VCP_PERF_VOLUME_PER_POD is not set") - volumesPerPod, err = strconv.Atoi(volumesPerPodStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_PERF_VOLUME_PER_POD") - - iterationsStr := os.Getenv("VCP_PERF_ITERATIONS") - Expect(iterationsStr).NotTo(BeEmpty(), "ENV VCP_PERF_ITERATIONS is not set") - iterations, err = strconv.Atoi(iterationsStr) - Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_PERF_ITERATIONS") - - policyName = os.Getenv("VSPHERE_SPBM_GOLD_POLICY") - datastoreName = os.Getenv("VSPHERE_DATASTORE") - Expect(policyName).NotTo(BeEmpty(), "ENV VSPHERE_SPBM_GOLD_POLICY is not set") - Expect(datastoreName).NotTo(BeEmpty(), "ENV VSPHERE_DATASTORE is not set") + policyName = GetAndExpectStringEnvVar(SPBMPolicyName) + datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName) nodes := framework.GetReadySchedulableNodesOrDie(client) Expect(len(nodes.Items)).To(BeNumerically(">=", 1), "Requires at least %d nodes (not %d)", 2, len(nodes.Items)) diff --git a/test/e2e/storage/vsphere/vsphere_volume_placement.go b/test/e2e/storage/vsphere/vsphere_volume_placement.go index a0327de9d4a..7695458b7eb 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_placement.go +++ b/test/e2e/storage/vsphere/vsphere_volume_placement.go @@ -18,7 +18,6 @@ package vsphere import ( "fmt" - "os" "strconv" "time" @@ -225,7 +224,7 @@ var _ = utils.SIGDescribe("Volume Placement", func() { volumeOptions = new(vclib.VolumeOptions) volumeOptions.CapacityKB = 2097152 volumeOptions.Name = "e2e-vmdk-" + strconv.FormatInt(time.Now().UnixNano(), 10) - volumeOptions.Datastore = os.Getenv("VSPHERE_SECOND_SHARED_DATASTORE") + volumeOptions.Datastore = GetAndExpectStringEnvVar(SecondSharedDatastore) volumePath, err := createVSphereVolume(vsp, volumeOptions) Expect(err).NotTo(HaveOccurred()) volumePaths = append(volumePaths, volumePath) diff --git a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go index 52c86a2e538..3c51cbce4eb 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go +++ b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go @@ -19,21 +19,16 @@ package vsphere import ( "fmt" "hash/fnv" - "os" "time" "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/vmware/govmomi/find" - "golang.org/x/net/context" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stype "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" - vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -98,11 +93,15 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp client clientset.Interface namespace string scParameters map[string]string + policyName string + tagPolicy string ) BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") client = f.ClientSet namespace = f.Namespace.Name + policyName = GetAndExpectStringEnvVar(SPBMPolicyName) + tagPolicy = GetAndExpectStringEnvVar(SPBMTagPolicy) framework.Logf("framework: %+v", f) scParameters = make(map[string]string) nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) @@ -113,47 +112,47 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp // Valid policy. It("verify VSAN storage capability with valid hostFailuresToTolerate and cacheReservation values is honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy hostFailuresToTolerate: %s, cacheReservation: %s", HostFailuresToTolerateCapabilityVal, CacheReservationCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy hostFailuresToTolerate: %s, cacheReservation: %s", HostFailuresToTolerateCapabilityVal, CacheReservationCapabilityVal)) scParameters[Policy_HostFailuresToTolerate] = HostFailuresToTolerateCapabilityVal scParameters[Policy_CacheReservation] = CacheReservationCapabilityVal - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) invokeValidPolicyTest(f, client, namespace, scParameters) }) // Valid policy. It("verify VSAN storage capability with valid diskStripes and objectSpaceReservation values is honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal)) scParameters[Policy_DiskStripes] = "1" scParameters[Policy_ObjectSpaceReservation] = "30" - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) invokeValidPolicyTest(f, client, namespace, scParameters) }) // Valid policy. It("verify VSAN storage capability with valid diskStripes and objectSpaceReservation values and a VSAN datastore is honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal)) scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal scParameters[Datastore] = VsanDatastore - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) invokeValidPolicyTest(f, client, namespace, scParameters) }) // Valid policy. It("verify VSAN storage capability with valid objectSpaceReservation and iopsLimit values is honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy objectSpaceReservation: %s, iopsLimit: %s", ObjectSpaceReservationCapabilityVal, IopsLimitCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy objectSpaceReservation: %s, iopsLimit: %s", ObjectSpaceReservationCapabilityVal, IopsLimitCapabilityVal)) scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal scParameters[Policy_IopsLimit] = IopsLimitCapabilityVal - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) invokeValidPolicyTest(f, client, namespace, scParameters) }) // Invalid VSAN storage capabilties parameters. It("verify VSAN storage capability with invalid capability name objectSpaceReserve is not honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy objectSpaceReserve: %s, stripeWidth: %s", ObjectSpaceReservationCapabilityVal, StripeWidthCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy objectSpaceReserve: %s, stripeWidth: %s", ObjectSpaceReservationCapabilityVal, StripeWidthCapabilityVal)) scParameters["objectSpaceReserve"] = ObjectSpaceReservationCapabilityVal scParameters[Policy_DiskStripes] = StripeWidthCapabilityVal - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "invalid option \\\"objectSpaceReserve\\\" for volume plugin kubernetes.io/vsphere-volume" @@ -165,10 +164,10 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp // Invalid policy on a VSAN test bed. // diskStripes value has to be between 1 and 12. It("verify VSAN storage capability with invalid diskStripes value is not honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, cacheReservation: %s", DiskStripesCapabilityInvalidVal, CacheReservationCapabilityVal)) + By(fmt.Sprintf("Invoking test for VSAN policy diskStripes: %s, cacheReservation: %s", DiskStripesCapabilityInvalidVal, CacheReservationCapabilityVal)) scParameters[Policy_DiskStripes] = DiskStripesCapabilityInvalidVal scParameters[Policy_CacheReservation] = CacheReservationCapabilityVal - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "Invalid value for " + Policy_DiskStripes + "." @@ -180,9 +179,9 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp // Invalid policy on a VSAN test bed. // hostFailuresToTolerate value has to be between 0 and 3 including. It("verify VSAN storage capability with invalid hostFailuresToTolerate value is not honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy hostFailuresToTolerate: %s", HostFailuresToTolerateCapabilityInvalidVal)) + By(fmt.Sprintf("Invoking test for VSAN policy hostFailuresToTolerate: %s", HostFailuresToTolerateCapabilityInvalidVal)) scParameters[Policy_HostFailuresToTolerate] = HostFailuresToTolerateCapabilityInvalidVal - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "Invalid value for " + Policy_HostFailuresToTolerate + "." @@ -194,11 +193,11 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp // Specify a valid VSAN policy on a non-VSAN test bed. // The test should fail. It("verify VSAN storage capability with non-vsan datastore is not honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s and a non-VSAN datastore: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal, VmfsDatastore)) + By(fmt.Sprintf("Invoking test for VSAN policy diskStripes: %s, objectSpaceReservation: %s and a non-VSAN datastore: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal, VmfsDatastore)) scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal scParameters[Datastore] = VmfsDatastore - framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters) + framework.Logf("Invoking test for VSAN storage capabilities: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "The specified datastore: \\\"" + VmfsDatastore + "\\\" is not a VSAN datastore. " + @@ -209,12 +208,10 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp }) It("verify an existing and compatible SPBM policy is honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for SPBM policy: %s", os.Getenv("VSPHERE_SPBM_GOLD_POLICY"))) - goldPolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY") - Expect(goldPolicy).NotTo(BeEmpty()) - scParameters[SpbmStoragePolicy] = goldPolicy + By(fmt.Sprintf("Invoking test for SPBM policy: %s", policyName)) + scParameters[SpbmStoragePolicy] = policyName scParameters[DiskFormat] = ThinDisk - framework.Logf("Invoking Test for SPBM storage policy: %+v", scParameters) + framework.Logf("Invoking test for SPBM storage policy: %+v", scParameters) invokeValidPolicyTest(f, client, namespace, scParameters) }) @@ -222,33 +219,30 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp scParameters[Policy_DiskStripes] = DiskStripesCapabilityMaxVal scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal scParameters[Datastore] = VsanDatastore - framework.Logf("Invoking Test for SPBM storage policy: %+v", scParameters) - clusterName := os.Getenv("VSPHERE_KUBERNETES_CLUSTER") - Expect(clusterName).NotTo(BeEmpty()) - invokeStaleDummyVMTestWithStoragePolicy(client, namespace, clusterName, scParameters) + framework.Logf("Invoking test for SPBM storage policy: %+v", scParameters) + kubernetesClusterName := GetAndExpectStringEnvVar(KubernetesClusterName) + invokeStaleDummyVMTestWithStoragePolicy(client, namespace, kubernetesClusterName, scParameters) }) It("verify if a SPBM policy is not honored on a non-compatible datastore for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for SPBM policy: %s and datastore: %s", os.Getenv("VSPHERE_SPBM_TAG_POLICY"), VsanDatastore)) - tagPolicy := os.Getenv("VSPHERE_SPBM_TAG_POLICY") - Expect(tagPolicy).NotTo(BeEmpty()) + By(fmt.Sprintf("Invoking test for SPBM policy: %s and datastore: %s", tagPolicy, VsanDatastore)) scParameters[SpbmStoragePolicy] = tagPolicy scParameters[Datastore] = VsanDatastore scParameters[DiskFormat] = ThinDisk - framework.Logf("Invoking Test for SPBM storage policy on a non-compatible datastore: %+v", scParameters) + framework.Logf("Invoking test for SPBM storage policy on a non-compatible datastore: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) - errorMsg := "User specified datastore is not compatible with the storagePolicy: \\\"" + os.Getenv("VSPHERE_SPBM_TAG_POLICY") + "\\\"" + errorMsg := "User specified datastore is not compatible with the storagePolicy: \\\"" + tagPolicy + "\\\"" if !strings.Contains(err.Error(), errorMsg) { Expect(err).NotTo(HaveOccurred(), errorMsg) } }) It("verify if a non-existing SPBM policy is not honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for SPBM policy: %s", BronzeStoragePolicy)) + By(fmt.Sprintf("Invoking test for SPBM policy: %s", BronzeStoragePolicy)) scParameters[SpbmStoragePolicy] = BronzeStoragePolicy scParameters[DiskFormat] = ThinDisk - framework.Logf("Invoking Test for non-existing SPBM storage policy: %+v", scParameters) + framework.Logf("Invoking test for non-existing SPBM storage policy: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "no pbm profile found with name: \\\"" + BronzeStoragePolicy + "\\" @@ -258,14 +252,12 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp }) It("verify an if a SPBM policy and VSAN capabilities cannot be honored for dynamically provisioned pvc using storageclass", func() { - By(fmt.Sprintf("Invoking Test for SPBM policy: %s with VSAN storage capabilities", os.Getenv("VSPHERE_SPBM_GOLD_POLICY"))) - goldPolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY") - Expect(goldPolicy).NotTo(BeEmpty()) - scParameters[SpbmStoragePolicy] = goldPolicy + By(fmt.Sprintf("Invoking test for SPBM policy: %s with VSAN storage capabilities", policyName)) + scParameters[SpbmStoragePolicy] = policyName Expect(scParameters[SpbmStoragePolicy]).NotTo(BeEmpty()) scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal scParameters[DiskFormat] = ThinDisk - framework.Logf("Invoking Test for SPBM storage policy and VSAN capabilities together: %+v", scParameters) + framework.Logf("Invoking test for SPBM storage policy and VSAN capabilities together: %+v", scParameters) err := invokeInvalidPolicyTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) errorMsg := "Cannot specify storage policy capabilities along with storage policy name. Please specify only one" @@ -355,23 +347,7 @@ func invokeStaleDummyVMTestWithStoragePolicy(client clientset.Interface, namespa fnvHash.Write([]byte(vmName)) dummyVMFullName := DummyVMPrefixName + "-" + fmt.Sprint(fnvHash.Sum32()) errorMsg := "Dummy VM - " + vmName + "is still present. Failing the test.." - Expect(isDummyVMPresent(dummyVMFullName)).NotTo(BeTrue(), errorMsg) -} - -func isDummyVMPresent(vmName string) bool { - By("Verifing if the dummy VM is deleted by the vSphere Cloud Provider clean up routine") - govMoMiClient, err := vsphere.GetgovmomiClient(nil) + vsp, err := getVSphere(client) Expect(err).NotTo(HaveOccurred()) - - f := find.NewFinder(govMoMiClient.Client, true) - ctx, _ := context.WithCancel(context.Background()) - - workingDir := os.Getenv("VSPHERE_WORKING_DIR") - Expect(workingDir).NotTo(BeEmpty()) - vmPath := workingDir + vmName - _, err = f.VirtualMachine(ctx, vmPath) - if err != nil { - return false - } - return true + Expect(vsp.IsDummyVMPresent(dummyVMFullName)).NotTo(BeTrue(), errorMsg) }