updating existing e2e tests and adding new e2e tests for vsphere cloud provider

addressed review comments

Addressed review comment for pv_reclaimpolicy.go to verify content of the volume

addressed 2nd round of review comments

addressed 3rd round of review comments from jeffvance

pkg/cloudprovider/providers/vsphere/BUILD

@@ -10,7 +10,10 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["vsphere.go"],
+    srcs = [
+        "vsphere.go",
+        "vsphere_util.go",
+    ],
     tags = ["automanaged"],
     deps = [
         "//pkg/api/v1:go_default_library",

pkg/cloudprovider/providers/vsphere/vsphere_util.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"strings"
+)
+
+// Reads vSphere configuration from system environment and construct vSphere object
+func GetVSphere() (*VSphere, error) {
+	var cfg VSphereConfig
+	var err error
+	cfg.Global.VCenterIP = os.Getenv("VSPHERE_VCENTER")
+	cfg.Global.VCenterPort = os.Getenv("VSPHERE_VCENTER_PORT")
+	cfg.Global.User = os.Getenv("VSPHERE_USER")
+	cfg.Global.Password = os.Getenv("VSPHERE_PASSWORD")
+	cfg.Global.Datacenter = os.Getenv("VSPHERE_DATACENTER")
+	cfg.Global.Datastore = os.Getenv("VSPHERE_DATASTORE")
+	cfg.Global.WorkingDir = os.Getenv("VSPHERE_WORKING_DIR")
+	cfg.Global.InsecureFlag = false
+	if strings.ToLower(os.Getenv("VSPHERE_INSECURE")) == "true" {
+		cfg.Global.InsecureFlag = true
+	}
+	c, err := newClient(context.TODO(), &cfg)
+	if err != nil {
+		return nil, err
+	}
+	vs := VSphere{
+		client:          c,
+		cfg:             &cfg,
+		localInstanceID: "",
+	}
+	runtime.SetFinalizer(&vs, logout)
+	return &vs, nil
+}

test/e2e/BUILD

@@ -69,11 +69,14 @@ go_library(
         "pd.go",
         "persistent_volumes.go",
         "persistent_volumes-disruptive.go",
+        "persistent_volumes-vsphere.go",
         "pod_gc.go",
         "pods.go",
         "portforward.go",
         "pre_stop.go",
         "proxy.go",
+        "pv_reclaimpolicy.go",
+        "pvc_label_selector.go",
         "pvutil.go",
         "rc.go",
         "reboot.go",
@@ -95,6 +98,8 @@ go_library(
         "util_iperf.go",
         "volume_provisioning.go",
         "volumes.go",
+        "vsphere_utils.go",
+        "vsphere_volume_placement.go",
     ],
     tags = ["automanaged"],
     deps = [
@@ -122,6 +127,7 @@ go_library(
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/aws:go_default_library",
         "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/deployment/util:go_default_library",
         "//pkg/controller/endpoint:go_default_library",

test/e2e/persistent_volumes-vsphere.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Testing configurations of single a PV/PVC pair attached to a vSphere Disk
+var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
+	var (
+		c          clientset.Interface
+		ns         string
+		volumePath string
+		pv         *v1.PersistentVolume
+		pvc        *v1.PersistentVolumeClaim
+		clientPod  *v1.Pod
+		pvConfig   persistentVolumeConfig
+		vsp        *vsphere.VSphere
+		err        error
+		node       types.NodeName
+	)
+
+	f := framework.NewDefaultFramework("pv")
+	/*
+		Test Setup
+
+		1. Create volume (vmdk)
+		2. Create PV with volume path for the vmdk.
+		3. Create PVC to bind with PV.
+		4. Create a POD using the PVC.
+		5. Verify Disk and Attached to the node.
+	*/
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		clientPod = nil
+		pvc = nil
+		pv = nil
+
+		if vsp == nil {
+			vsp, err = vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+		}
+		if volumePath == "" {
+			volumePath, err = createVSphereVolume(vsp, nil)
+			Expect(err).NotTo(HaveOccurred())
+			pvConfig = persistentVolumeConfig{
+				namePrefix: "vspherepv-",
+				pvSource: v1.PersistentVolumeSource{
+					VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+						VolumePath: volumePath,
+						FSType:     "ext4",
+					},
+				},
+				prebind: nil,
+			}
+		}
+		By("Creating the PV and PVC")
+		pv, pvc = createPVPVC(c, pvConfig, ns, false)
+		waitOnPVandPVC(c, ns, pv, pvc)
+
+		By("Creating the Client Pod")
+		clientPod = createClientPod(c, ns, pvc)
+		node := types.NodeName(clientPod.Spec.NodeName)
+
+		By("Verify disk should be attached to the node")
+		isAttached, err := verifyVSphereDiskAttached(vsp, volumePath, node)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(isAttached).To(BeTrue(), "disk is not attached with the node")
+	})
+
+	AfterEach(func() {
+		framework.Logf("AfterEach: Cleaning up test resources")
+		if c != nil {
+			if clientPod != nil {
+				clientPod, err = c.CoreV1().Pods(ns).Get(clientPod.Name, metav1.GetOptions{})
+				if !apierrs.IsNotFound(err) {
+					deletePodWithWait(f, c, clientPod)
+				}
+			}
+
+			if pv != nil {
+				deletePersistentVolume(c, pv.Name)
+			}
+			if pvc != nil {
+				deletePersistentVolumeClaim(c, pvc.Name, ns)
+			}
+		}
+	})
+	/*
+		Clean up
+
+		1. Wait and verify volume is detached from the node
+		2. Delete PV
+		3. Delete Volume (vmdk)
+	*/
+	AddCleanupAction(func() {
+		if len(volumePath) > 0 {
+			waitForVSphereDiskToDetach(vsp, volumePath, node)
+			vsp.DeleteVolume(volumePath)
+		}
+	})
+
+	/*
+		Delete the PVC and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+
+		Test Steps:
+		1. Delete PVC.
+		2. Delete POD, POD deletion should succeed.
+	*/
+
+	It("should test that deleting a PVC before the pod does not cause pod deletion to fail on PD detach", func() {
+		By("Deleting the Claim")
+		deletePersistentVolumeClaim(c, pvc.Name, ns)
+
+		pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+		if !apierrs.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		pvc = nil
+		By("Deleting the Pod")
+		deletePodWithWait(f, c, clientPod)
+
+	})
+
+	/*
+		Delete the PV and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+
+		Test Steps:
+		1. Delete PV.
+		2. Delete POD, POD deletion should succeed.
+	*/
+	It("should test that deleting the PV before the pod does not cause pod deletion to fail on PD detach", func() {
+		By("Deleting the Persistent Volume")
+		deletePersistentVolume(c, pv.Name)
+		pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+		if !apierrs.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		pv = nil
+		By("Deleting the pod")
+		deletePodWithWait(f, c, clientPod)
+	})
+})

test/e2e/pv_reclaimpolicy.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = framework.KubeDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() {
+	f := framework.NewDefaultFramework("persistentvolumereclaim")
+	var (
+		c          clientset.Interface
+		ns         string
+		volumePath string
+		pv         *v1.PersistentVolume
+		pvc        *v1.PersistentVolumeClaim
+	)
+
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+	})
+
+	framework.KubeDescribe("persistentvolumereclaim:vsphere", func() {
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("vsphere")
+			pv = nil
+			pvc = nil
+			volumePath = ""
+		})
+
+		AfterEach(func() {
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+			testCleanupVSpherePersistentVolumeReclaim(vsp, c, ns, volumePath, pv, pvc)
+		})
+
+		/*
+			This test verifies persistent volume should be deleted when reclaimPolicy on the PV is set to delete and
+			associated claim is deleted
+
+			Test Steps:
+			1. Create vmdk
+			2. Create PV Spec with volume path set to VMDK file created in Step-1, and PersistentVolumeReclaimPolicy is set to Delete
+			3. Create PVC with the storage request set to PV's storage capacity.
+			4. Wait for PV and PVC to bound.
+			5. Delete PVC
+			6. Verify PV is deleted automatically.
+		*/
+		It("should delete persistent volume when reclaimPolicy set to delete and associated claim is deleted", func() {
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, pv, pvc, err = testSetupVSpherePersistentVolumeReclaim(vsp, c, ns, v1.PersistentVolumeReclaimDelete)
+			Expect(err).NotTo(HaveOccurred())
+
+			deletePVCAfterBind(c, ns, pvc, pv)
+			pvc = nil
+
+			By("verify pv is deleted")
+			err = framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			pv = nil
+			volumePath = ""
+		})
+
+		/*
+			This test Verify persistent volume should be retained when reclaimPolicy on the PV is set to retain
+			and associated claim is deleted
+
+			Test Steps:
+			1. Create vmdk
+			2. Create PV Spec with volume path set to VMDK file created in Step-1, and PersistentVolumeReclaimPolicy is set to Retain
+			3. Create PVC with the storage request set to PV's storage capacity.
+			4. Wait for PV and PVC to bound.
+			5. Write some content in the volume.
+			6. Delete PVC
+			7. Verify PV is retained.
+			8. Delete retained PV.
+			9. Create PV Spec with the same volume path used in step 2.
+			10. Create PVC with the storage request set to PV's storage capacity.
+			11. Created POD using PVC created in Step 10 and verify volume content is matching.
+		*/
+
+		It("should retain persistent volume when reclaimPolicy set to retain when associated claim is deleted", func() {
+			var volumeFileContent = "hello from vsphere cloud provider, Random Content is :" + strconv.FormatInt(time.Now().UnixNano(), 10)
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, pv, pvc, err = testSetupVSpherePersistentVolumeReclaim(vsp, c, ns, v1.PersistentVolumeReclaimRetain)
+			Expect(err).NotTo(HaveOccurred())
+
+			writeContentToVSpherePV(c, pvc, volumeFileContent)
+
+			By("Delete PVC")
+			deletePersistentVolumeClaim(c, pvc.Name, ns)
+			pvc = nil
+
+			By("Verify PV is retained")
+			framework.Logf("Waiting for PV %v to become Released", pv.Name)
+			err = framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			deletePersistentVolume(c, pv.Name)
+
+			By("Creating the PV for same volume path")
+			pv = getVSpherePersistentVolumeSpec(volumePath, v1.PersistentVolumeReclaimRetain, nil)
+			pv, err = c.CoreV1().PersistentVolumes().Create(pv)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("creating the pvc")
+			pvc = getVSpherePersistentVolumeClaimSpec(ns, nil)
+			pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("wait for the pv and pvc to bind")
+			waitOnPVandPVC(c, ns, pv, pvc)
+			verifyContentOfVSpherePV(c, pvc, volumeFileContent)
+
+		})
+	})
+})
+
+// Test Setup for persistentvolumereclaim tests for vSphere Provider
+func testSetupVSpherePersistentVolumeReclaim(vsp *vsphere.VSphere, c clientset.Interface, ns string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy) (volumePath string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim, err error) {
+	By("running testSetupVSpherePersistentVolumeReclaim")
+	By("creating vmdk")
+	volumePath, err = createVSphereVolume(vsp, nil)
+	if err != nil {
+		return
+	}
+	By("creating the pv")
+	pv = getVSpherePersistentVolumeSpec(volumePath, persistentVolumeReclaimPolicy, nil)
+	pv, err = c.CoreV1().PersistentVolumes().Create(pv)
+	if err != nil {
+		return
+	}
+	By("creating the pvc")
+	pvc = getVSpherePersistentVolumeClaimSpec(ns, nil)
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+	return
+}
+
+// Test Cleanup for persistentvolumereclaim tests for vSphere Provider
+func testCleanupVSpherePersistentVolumeReclaim(vsp *vsphere.VSphere, c clientset.Interface, ns string, volumePath string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) {
+	By("running testCleanupVSpherePersistentVolumeReclaim")
+	if len(volumePath) > 0 {
+		vsp.DeleteVolume(volumePath)
+	}
+	if pv != nil {
+		deletePersistentVolume(c, pv.Name)
+	}
+	if pvc != nil {
+		deletePersistentVolumeClaim(c, pvc.Name, ns)
+	}
+}
+
+// func to wait until PV and PVC bind and once bind completes, delete the PVC
+func deletePVCAfterBind(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	var err error
+
+	By("wait for the pv and pvc to bind")
+	waitOnPVandPVC(c, ns, pv, pvc)
+
+	By("delete pvc")
+	deletePersistentVolumeClaim(c, pvc.Name, ns)
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+	if !apierrs.IsNotFound(err) {
+		Expect(err).NotTo(HaveOccurred())
+	}
+}

test/e2e/pvc_label_selector.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+/*
+   This is a function test for Selector-Label Volume Binding Feature
+   Test verifies volume with the matching label is bounded with the PVC.
+
+   Test Steps
+   ----------
+   1. Create VMDK.
+   2. Create pv with lable volume-type:ssd, volume path set to vmdk created in previous step, and PersistentVolumeReclaimPolicy is set to Delete.
+   3. Create PVC (pvc_vvol) with label selector to match with volume-type:vvol
+   4. Create PVC (pvc_ssd) with label selector to match with volume-type:ssd
+   5. Wait and verify pvc_ssd is bound with PV.
+   6. Verify Status of pvc_vvol is still pending.
+   7. Delete pvc_ssd.
+   8. verify associated pv is also deleted.
+   9. delete pvc_vvol
+
+*/
+var _ = framework.KubeDescribe("PersistentVolumes [Feature:LabelSelector]", func() {
+	f := framework.NewDefaultFramework("pvclabelselector")
+	var (
+		c          clientset.Interface
+		ns         string
+		pv_ssd     *v1.PersistentVolume
+		pvc_ssd    *v1.PersistentVolumeClaim
+		pvc_vvol   *v1.PersistentVolumeClaim
+		volumePath string
+		ssdlabels  map[string]string
+		vvollabels map[string]string
+		err        error
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+		ssdlabels = make(map[string]string)
+		ssdlabels["volume-type"] = "ssd"
+		vvollabels = make(map[string]string)
+		vvollabels["volume-type"] = "vvol"
+
+	})
+
+	framework.KubeDescribe("Selector-Label Volume Binding:vsphere", func() {
+		AfterEach(func() {
+			By("Running clean up actions")
+			if framework.ProviderIs("vsphere") {
+				testCleanupVSpherePVClabelselector(c, ns, volumePath, pv_ssd, pvc_ssd, pvc_vvol)
+			}
+		})
+		It("should bind volume with claim for given label", func() {
+			volumePath, pv_ssd, pvc_ssd, pvc_vvol, err = testSetupVSpherePVClabelselector(c, ns, ssdlabels, vvollabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("wait for the pvc_ssd to bind with pv_ssd")
+			waitOnPVandPVC(c, ns, pv_ssd, pvc_ssd)
+
+			By("Verify status of pvc_vvol is pending")
+			err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimPending, c, ns, pvc_vvol.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("delete pvc_ssd")
+			deletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
+
+			By("verify pv_ssd is deleted")
+			err = framework.WaitForPersistentVolumeDeleted(c, pv_ssd.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			volumePath = ""
+
+			By("delete pvc_vvol")
+			deletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
+		})
+	})
+})
+
+func testSetupVSpherePVClabelselector(c clientset.Interface, ns string, ssdlabels map[string]string, vvollabels map[string]string) (volumePath string, pv_ssd *v1.PersistentVolume, pvc_ssd *v1.PersistentVolumeClaim, pvc_vvol *v1.PersistentVolumeClaim, err error) {
+	volumePath = ""
+	By("creating vmdk")
+	vsp, err := vsphere.GetVSphere()
+	Expect(err).NotTo(HaveOccurred())
+	volumePath, err = createVSphereVolume(vsp, nil)
+	if err != nil {
+		return
+	}
+
+	By("creating the pv with lable volume-type:ssd")
+	pv_ssd = getVSpherePersistentVolumeSpec(volumePath, v1.PersistentVolumeReclaimDelete, ssdlabels)
+	pv_ssd, err = c.CoreV1().PersistentVolumes().Create(pv_ssd)
+	if err != nil {
+		return
+	}
+
+	By("creating pvc with label selector to match with volume-type:vvol")
+	pvc_vvol = getVSpherePersistentVolumeClaimSpec(ns, vvollabels)
+	pvc_vvol, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc_vvol)
+	if err != nil {
+		return
+	}
+
+	By("creating pvc with label selector to match with volume-type:ssd")
+	pvc_ssd = getVSpherePersistentVolumeClaimSpec(ns, ssdlabels)
+	pvc_ssd, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc_ssd)
+	return
+}
+
+func testCleanupVSpherePVClabelselector(c clientset.Interface, ns string, volumePath string, pv_ssd *v1.PersistentVolume, pvc_ssd *v1.PersistentVolumeClaim, pvc_vvol *v1.PersistentVolumeClaim) {
+	By("running testCleanupVSpherePVClabelselector")
+	if len(volumePath) > 0 {
+		vsp, err := vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+		vsp.DeleteVolume(volumePath)
+	}
+	if pvc_ssd != nil {
+		deletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
+	}
+	if pvc_vvol != nil {
+		deletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
+	}
+	if pv_ssd != nil {
+		deletePersistentVolume(c, pv_ssd.Name)
+	}
+}

test/e2e/volume_provisioning.go

@@ -34,9 +34,10 @@ import (
 const (
 	// Requested size of the volume
 	requestedSize = "1500Mi"
-	// Expected size of the volume is 2GiB, because all three supported cloud
+	// Expected size of the volume is 2GiB, for "openstack", "gce", "aws", "gke", as they allocate volumes in 1GiB chunks
-	// providers allocate volumes in 1GiB chunks.
 	expectedSize = "2Gi"
+	// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size equal to requestedSize
+	vsphereExpectedSize = "1500Mi"
 )
 
 func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim) {
@@ -54,12 +55,15 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol
 
 	// Check sizes
 	expectedCapacity := resource.MustParse(expectedSize)
+	if framework.ProviderIs("vsphere") {
+		expectedCapacity = resource.MustParse(vsphereExpectedSize)
+	}
 	pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
-	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()))
+	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")
 
 	requestedCapacity := resource.MustParse(requestedSize)
 	claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
-	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()))
+	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")
 
 	// Check PV properties
 	Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
@@ -103,7 +107,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 
 	framework.KubeDescribe("DynamicProvisioner", func() {
 		It("should create and delete persistent volumes [Slow] [Volume]", func() {
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
+			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
 
 			By("creating a StorageClass")
 			class := newStorageClass()
@@ -125,7 +129,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 
 	framework.KubeDescribe("DynamicProvisioner Alpha", func() {
 		It("should create and delete alpha persistent volumes [Slow] [Volume]", func() {
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
+			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
 
 			By("creating a claim with an alpha dynamic provisioning annotation")
 			claim := newClaim(ns, true)
@@ -229,6 +233,8 @@ func newStorageClass() *storage.StorageClass {
 		pluginName = "kubernetes.io/aws-ebs"
 	case framework.ProviderIs("openstack"):
 		pluginName = "kubernetes.io/cinder"
+	case framework.ProviderIs("vsphere"):
+		pluginName = "kubernetes.io/vsphere-volume"
 	}
 
 	return &storage.StorageClass{

test/e2e/volumes.go

@@ -50,6 +50,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	"github.com/golang/glog"
@@ -928,4 +929,58 @@ var _ = framework.KubeDescribe("Volumes [Feature:Volumes]", func() {
 			testVolumeClient(cs, config, nil, tests)
 		})
 	})
+
+	////////////////////////////////////////////////////////////////////////
+	// vSphere
+	////////////////////////////////////////////////////////////////////////
+
+	framework.KubeDescribe("vsphere", func() {
+		It("should be mountable", func() {
+			framework.SkipUnlessProviderIs("vsphere")
+			var (
+				volumePath string
+			)
+			config := VolumeTestConfig{
+				namespace: namespace.Name,
+				prefix:    "vsphere",
+			}
+			By("creating a test vsphere volume")
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, err = createVSphereVolume(vsp, nil)
+			Expect(err).NotTo(HaveOccurred())
+
+			defer func() {
+				vsp.DeleteVolume(volumePath)
+			}()
+
+			defer func() {
+				if clean {
+					framework.Logf("Running volumeTestCleanup")
+					volumeTestCleanup(f, config)
+				}
+			}()
+
+			tests := []VolumeTest{
+				{
+					volume: v1.VolumeSource{
+						VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+							VolumePath: volumePath,
+							FSType:     "ext4",
+						},
+					},
+					file: "index.html",
+					// Randomize index.html to make sure we don't see the
+					// content from previous test runs.
+					expectedContent: "Hello from vSphere from namespace " + namespace.Name,
+				},
+			}
+
+			injectHtml(cs, config, tests[0].volume, tests[0].expectedContent)
+
+			fsGroup := int64(1234)
+			testVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
 })

test/e2e/vsphere_utils.go

@@ -0,0 +1,180 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"strconv"
+	"time"
+
+	"fmt"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// Sanity check for vSphere testing.  Verify the persistent disk attached to the node.
+func verifyVSphereDiskAttached(vsp *vsphere.VSphere, volumePath string, nodeName types.NodeName) (bool, error) {
+	var (
+		isAttached bool
+		err        error
+	)
+	if vsp == nil {
+		vsp, err = vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+	}
+	isAttached, err = vsp.DiskIsAttached(volumePath, nodeName)
+	Expect(err).NotTo(HaveOccurred())
+	return isAttached, err
+}
+
+// Wait until vsphere vmdk is deteched from the given node or time out after 5 minutes
+func waitForVSphereDiskToDetach(vsp *vsphere.VSphere, volumePath string, nodeName types.NodeName) {
+	var (
+		err            error
+		diskAttached   = true
+		detachTimeout  = 5 * time.Minute
+		detachPollTime = 10 * time.Second
+	)
+	if vsp == nil {
+		vsp, err = vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+	}
+	err = wait.Poll(detachPollTime, detachTimeout, func() (bool, error) {
+		diskAttached, err = verifyVSphereDiskAttached(vsp, volumePath, nodeName)
+		if err != nil {
+			return true, err
+		}
+		if !diskAttached {
+			framework.Logf("Volume %q appears to have successfully detached from %q.",
+				volumePath, nodeName)
+			return true, nil
+		}
+		framework.Logf("Waiting for Volume %q to detach from %q.", volumePath, nodeName)
+		return false, nil
+	})
+	Expect(err).NotTo(HaveOccurred())
+	if diskAttached {
+		Expect(fmt.Errorf("Gave up waiting for Volume %q to detach from %q after %v", volumePath, nodeName, detachTimeout)).NotTo(HaveOccurred())
+	}
+
+}
+
+// function to create vsphere volume spec with given VMDK volume path, Reclaim Policy and labels
+func getVSpherePersistentVolumeSpec(volumePath string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy, labels map[string]string) *v1.PersistentVolume {
+	var (
+		pvConfig persistentVolumeConfig
+		pv       *v1.PersistentVolume
+		claimRef *v1.ObjectReference
+	)
+	pvConfig = persistentVolumeConfig{
+		namePrefix: "vspherepv-",
+		pvSource: v1.PersistentVolumeSource{
+			VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+				VolumePath: volumePath,
+				FSType:     "ext4",
+			},
+		},
+		prebind: nil,
+	}
+
+	pv = &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: pvConfig.namePrefix,
+			Annotations: map[string]string{
+				volumehelper.VolumeGidAnnotationKey: "777",
+			},
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeReclaimPolicy: persistentVolumeReclaimPolicy,
+			Capacity: v1.ResourceList{
+				v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+			},
+			PersistentVolumeSource: pvConfig.pvSource,
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			ClaimRef: claimRef,
+		},
+	}
+	if labels != nil {
+		pv.Labels = labels
+	}
+	return pv
+}
+
+// function to get vsphere persistent volume spec with given selector labels.
+func getVSpherePersistentVolumeClaimSpec(namespace string, labels map[string]string) *v1.PersistentVolumeClaim {
+	var (
+		pvc *v1.PersistentVolumeClaim
+	)
+	pvc = &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-",
+			Namespace:    namespace,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+				},
+			},
+		},
+	}
+	if labels != nil {
+		pvc.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels}
+	}
+
+	return pvc
+}
+
+// function to create vmdk volume
+func createVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vsphere.VolumeOptions) (string, error) {
+	var (
+		volumePath string
+		err        error
+	)
+	if volumeOptions == nil {
+		volumeOptions = new(vsphere.VolumeOptions)
+		volumeOptions.CapacityKB = 2097152
+		volumeOptions.Name = "e2e-vmdk-" + strconv.FormatInt(time.Now().UnixNano(), 10)
+	}
+	volumePath, err = vsp.CreateVolume(volumeOptions)
+	Expect(err).NotTo(HaveOccurred())
+	return volumePath, nil
+}
+
+// function to write content to the volume backed by given PVC
+func writeContentToVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
+	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data")
+	framework.Logf("Done with writing content to volume")
+}
+
+// function to verify content is matching on the volume backed for given PVC
+func verifyContentOfVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
+	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data")
+	framework.Logf("Sucessfully verified content of the volume")
+}

test/e2e/vsphere_volume_placement.go

@@ -0,0 +1,236 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = framework.KubeDescribe("Volume Placement [Feature:Volume]", func() {
+	f := framework.NewDefaultFramework("volume-placement")
+	var (
+		c                  clientset.Interface
+		ns                 string
+		vsp                *vsphere.VSphere
+		volumePath         string
+		node1Name          string
+		node1LabelValue    string
+		node1KeyValueLabel map[string]string
+
+		node2Name          string
+		node2LabelValue    string
+		node2KeyValueLabel map[string]string
+
+		isNodeLabeled bool
+	)
+
+	/*
+		Steps
+		1. Create VMDK volume
+		2. Find two nodes with the status available and ready for scheduling.
+		3. Add labels to the both nodes. - (vsphere_e2e_label: Random UUID)
+
+	*/
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+
+		By("creating vmdk")
+		vsp, err := vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+
+		volumePath, err = createVSphereVolume(vsp, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		if !isNodeLabeled {
+			nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+			if len(nodeList.Items) != 0 {
+				node1Name = nodeList.Items[0].Name
+				node2Name = nodeList.Items[1].Name
+			} else {
+				framework.Failf("Unable to find ready and schedulable Node")
+			}
+			node1LabelValue = "vsphere_e2e_" + string(uuid.NewUUID())
+			node1KeyValueLabel = make(map[string]string)
+			node1KeyValueLabel["vsphere_e2e_label"] = node1LabelValue
+			framework.AddOrUpdateLabelOnNode(c, node1Name, "vsphere_e2e_label", node1LabelValue)
+
+			node2LabelValue = "vsphere_e2e_" + string(uuid.NewUUID())
+			node2KeyValueLabel = make(map[string]string)
+			node2KeyValueLabel["vsphere_e2e_label"] = node2LabelValue
+			framework.AddOrUpdateLabelOnNode(c, node2Name, "vsphere_e2e_label", node2LabelValue)
+
+		}
+
+	})
+
+	/*
+		Steps
+		1. Remove labels assigned to node 1 and node 2
+		2. Delete VMDK volume
+	*/
+
+	AddCleanupAction(func() {
+		if len(node1LabelValue) > 0 {
+			framework.RemoveLabelOffNode(c, node1Name, "vsphere_e2e_label")
+		}
+		if len(node2LabelValue) > 0 {
+			framework.RemoveLabelOffNode(c, node2Name, "vsphere_e2e_label")
+		}
+		if len(volumePath) > 0 {
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+			vsp.DeleteVolume(volumePath)
+		}
+	})
+
+	framework.KubeDescribe("provision pod on node with matching labels", func() {
+
+		/*
+			Steps
+
+			1. Create POD Spec with volume path of the vmdk and NodeSelector set to label assigned to node1.
+			2. Create POD and wait for POD to become ready.
+			3. Verify volume is attached to the node1.
+			4. Delete POD.
+			5. Wait for volume to be detached from the node1.
+			6. Repeat Step 1 to 5 and make sure back to back pod creation on same worker node with the same volume is working as expected.
+
+		*/
+
+		It("should create and delete pod with the same volume source on the same worker node", func() {
+			pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePath)
+			deletePodAndWaitForVolumeToDetach(c, ns, vsp, node1Name, pod, volumePath)
+
+			By("Creating pod on the same node: " + node1Name)
+			pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePath)
+			deletePodAndWaitForVolumeToDetach(c, ns, vsp, node1Name, pod, volumePath)
+		})
+
+		/*
+			Steps
+
+			1. Create POD Spec with volume path of the vmdk and NodeSelector set to node1's label.
+			2. Create POD and wait for POD to become ready.
+			3. Verify volume is attached to the node1.
+			4. Delete POD.
+			5. Wait for volume to be detached from the node1.
+			6. Create POD Spec with volume path of the vmdk and NodeSelector set to node2's label.
+			7. Create POD and wait for POD to become ready.
+			8. Verify volume is attached to the node2.
+			9. Delete POD.
+		*/
+
+		It("should create and delete pod with the same volume source attach/detach to different worker nodes", func() {
+			pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePath)
+			deletePodAndWaitForVolumeToDetach(c, ns, vsp, node1Name, pod, volumePath)
+
+			By("Creating pod on the another node: " + node2Name)
+			pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node2Name, node2KeyValueLabel, volumePath)
+			deletePodAndWaitForVolumeToDetach(c, ns, vsp, node2Name, pod, volumePath)
+		})
+
+	})
+})
+
+func createPodWithVolumeAndNodeSelector(client clientset.Interface, namespace string, vsp *vsphere.VSphere, nodeName string, nodeKeyValueLabel map[string]string, volumePath string) *v1.Pod {
+	var pod *v1.Pod
+	var err error
+	By("Creating pod on the node: " + nodeName)
+	podspec := getPodSpec(volumePath, nodeKeyValueLabel, nil)
+
+	pod, err = client.CoreV1().Pods(namespace).Create(podspec)
+	Expect(err).NotTo(HaveOccurred())
+	By("Waiting for pod to be ready")
+	Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed())
+
+	By("Verify volume is attached to the node: " + nodeName)
+	isAttached, err := verifyVSphereDiskAttached(vsp, volumePath, types.NodeName(nodeName))
+	Expect(err).NotTo(HaveOccurred())
+	Expect(isAttached).To(BeTrue(), "disk is not attached with the node")
+	return pod
+}
+func deletePodAndWaitForVolumeToDetach(client clientset.Interface, namespace string, vsp *vsphere.VSphere, nodeName string, pod *v1.Pod, volumePath string) {
+	var err error
+	By("Deleting pod")
+	err = client.CoreV1().Pods(namespace).Delete(pod.Name, nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Waiting for volume to be detached from the node")
+	waitForVSphereDiskToDetach(vsp, volumePath, types.NodeName(nodeName))
+}
+
+func getPodSpec(volumePath string, keyValuelabel map[string]string, commands []string) *v1.Pod {
+	if commands == nil || len(commands) == 0 {
+		commands = make([]string, 3)
+		commands[0] = "/bin/sh"
+		commands[1] = "-c"
+		commands[2] = "while true ; do sleep 2 ; done "
+	}
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "vsphere-e2e-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "vsphere-e2e-container-" + string(uuid.NewUUID()),
+					Image:   "gcr.io/google_containers/busybox:1.24",
+					Command: commands,
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "vsphere-volume",
+							MountPath: "/mnt/vsphere-volume",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name: "vsphere-volume",
+					VolumeSource: v1.VolumeSource{
+						VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+							VolumePath: volumePath,
+							FSType:     "ext4",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if keyValuelabel != nil {
+		pod.Spec.NodeSelector = keyValuelabel
+	}
+	return pod
+}