Use any host that mounts the datastore to create Volume

Also, This change makes zone to work per datacenter and cleans up dummy vms.
There can be multiple datastores found for a given name. The datastore name is
unique only within a datacenter. So this commit returns a list of datastores
for a given datastore name in FindDatastoreByName() method. The calles are
responsible to handle or find the right datastore to use among those returned.

staging/src/k8s.io/legacy-cloud-providers/vsphere/BUILD

@@ -33,6 +33,7 @@ go_library(
         "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers:go_default_library",
+        "//vendor/github.com/vmware/govmomi/find:go_default_library",
         "//vendor/github.com/vmware/govmomi/object:go_default_library",
         "//vendor/github.com/vmware/govmomi/property:go_default_library",
         "//vendor/github.com/vmware/govmomi/vapi/rest:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/vsphere/nodemanager.go

@@ -39,6 +39,11 @@ type NodeInfo struct {
 	zone       *cloudprovider.Zone
 }
 
+func (n NodeInfo) String() string {
+	return fmt.Sprintf("{datacenter: %v, vm: %v, vcServer: %s, vmUUID: %s, zone: %v}",
+		*n.dataCenter, n.vm.Reference(), n.vcServer, n.vmUUID, *n.zone)
+}
+
 type NodeManager struct {
 	// TODO: replace map with concurrent map when k8s supports go v1.9
 

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/constants.go

@@ -52,6 +52,7 @@ const (
 	DatacenterType             = "Datacenter"
 	ClusterComputeResourceType = "ClusterComputeResource"
 	HostSystemType             = "HostSystem"
+	NameProperty               = "name"
 )
 
 // Test Constants

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/datacenter.go

@@ -187,6 +187,28 @@ func (dc *Datacenter) GetDatastoreByName(ctx context.Context, name string) (*Dat
 	return &datastore, nil
 }
 
+// GetDatastoreInfoByName gets the Datastore object for the given datastore name
+func (dc *Datacenter) GetDatastoreInfoByName(ctx context.Context, name string) (*DatastoreInfo, error) {
+	finder := getFinder(dc)
+	ds, err := finder.Datastore(ctx, name)
+	if err != nil {
+		klog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err)
+		return nil, err
+	}
+	datastore := Datastore{ds, dc}
+	var dsMo mo.Datastore
+	pc := property.DefaultCollector(dc.Client())
+	properties := []string{DatastoreInfoProperty}
+	err = pc.RetrieveOne(ctx, ds.Reference(), properties, &dsMo)
+	if err != nil {
+		klog.Errorf("Failed to get Datastore managed objects from datastore reference."+
+			" dsRef: %+v, err: %+v", ds.Reference(), err)
+		return nil, err
+	}
+	klog.V(9).Infof("Result dsMo: %+v", dsMo)
+	return &DatastoreInfo{Datastore: &datastore, Info: dsMo.Info.GetDatastoreInfo()}, nil
+}
+
 // GetResourcePool gets the resource pool for the given path
 func (dc *Datacenter) GetResourcePool(ctx context.Context, resourcePoolPath string) (*object.ResourcePool, error) {
 	finder := getFinder(dc)

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/datastore.go

@@ -96,8 +96,14 @@ func (ds *Datastore) GetDatastoreHostMounts(ctx context.Context) ([]types.Manage
 		return nil, err
 	}
 	hosts := make([]types.ManagedObjectReference, len(dsMo.Host))
-	for _, dsHostMount := range dsMo.Host {
+	for i, dsHostMount := range dsMo.Host {
-		hosts = append(hosts, dsHostMount.Key)
+		hosts[i] = dsHostMount.Key
 	}
 	return hosts, nil
 }
+
+// Exists returns whether the given file exists in this datastore
+func (ds *Datastore) Exists(ctx context.Context, file string) bool {
+	_, err := ds.Datastore.Stat(ctx, file)
+	return err == nil
+}

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers/vmdm.go

@@ -212,7 +212,7 @@ func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib
 }
 
 // CleanUpDummyVMs deletes stale dummyVM's
-func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error {
+func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder) error {
 	vmList, err := folder.GetVirtualMachines(ctx)
 	if err != nil {
 		klog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err)
@@ -231,7 +231,7 @@ func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datace
 			continue
 		}
 		if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) {
-			vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc}
+			vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(folder.Client(), vm.Reference())}
 			dummyVMList = append(dummyVMList, &vmObj)
 		}
 	}

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/pbm.go

@@ -85,7 +85,7 @@ func (pbmClient *PbmClient) IsDatastoreCompatible(ctx context.Context, storagePo
 
 // GetCompatibleDatastores filters and returns compatible list of datastores for given storage policy id
 // For Non Compatible Datastores, fault message with the Datastore Name is also returned
-func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, dc *Datacenter, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) {
+func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) {
 	var (
 		dsMorNameMap                                = getDsMorNameMap(ctx, datastores)
 		localizedMessagesForNotCompatibleDatastores = ""

staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/virtualmachine.go

@@ -233,7 +233,7 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da
 
 	var dsMoList []mo.Datastore
 	pc := property.DefaultCollector(vm.Client())
-	properties := []string{DatastoreInfoProperty}
+	properties := []string{DatastoreInfoProperty, NameProperty}
 	err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList)
 	if err != nil {
 		klog.Errorf("Failed to get Datastore managed objects from datastore objects."+
@@ -241,12 +241,19 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da
 		return nil, err
 	}
 	klog.V(9).Infof("Result dsMoList: %+v", dsMoList)
+	finder := getFinder(vm.Datacenter)
 	var dsObjList []*DatastoreInfo
 	for _, dsMo := range dsMoList {
+		// use the finder so that InventoryPath is set correctly in ds
+		ds, err := finder.Datastore(ctx, dsMo.Name)
+		if err != nil {
+			klog.Errorf("Failed finding datastore: %s. err: %+v", dsMo.Name, err)
+			return nil, err
+		}
+		datastore := Datastore{ds, vm.Datacenter}
 		dsObjList = append(dsObjList,
 			&DatastoreInfo{
-				&Datastore{object.NewDatastore(vm.Client(), dsMo.Reference()),
+				&datastore,
-					vm.Datacenter},
 				dsMo.Info.GetDatastoreInfo()})
 	}
 	return dsObjList, nil

staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go

@@ -1165,15 +1165,9 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string)
 func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) {
 	klog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions)
 	createVolumeInternal := func(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) {
-		var datastore string
+		var datastoreInfo *vclib.DatastoreInfo
 		var dsList []*vclib.DatastoreInfo
-		// If datastore not specified, then use default datastore
+
-		if volumeOptions.Datastore == "" {
-			datastore = vs.cfg.Workspace.DefaultDatastore
-		} else {
-			datastore = volumeOptions.Datastore
-		}
-		datastore = strings.TrimSpace(datastore)
 		// Create context
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
@@ -1181,16 +1175,29 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 		if err != nil {
 			return "", err
 		}
-		dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
+		// If datastore not specified, then use default datastore
+		datastoreName := strings.TrimSpace(volumeOptions.Datastore)
+		if datastoreName == "" {
+			datastoreName = strings.TrimSpace(vs.cfg.Workspace.DefaultDatastore)
+		}
+		// The given datastoreName may be present in more than one datacenter
+		candidateDatastoreInfos, err := vs.FindDatastoreByName(ctx, datastoreName)
 		if err != nil {
 			return "", err
 		}
+		// Each of the datastores found is a candidate for Volume creation.
+		// One of these will be selected based on given policy and/or zone.
+		candidateDatastores := make(map[string]*vclib.DatastoreInfo)
+		for _, dsInfo := range candidateDatastoreInfos {
+			candidateDatastores[dsInfo.Info.Url] = dsInfo
+		}
+
 		var vmOptions *vclib.VMOptions
 		if volumeOptions.VSANStorageProfileData != "" || volumeOptions.StoragePolicyName != "" {
 			// If datastore and zone are specified, first validate if the datastore is in the provided zone.
 			if len(volumeOptions.Zone) != 0 && volumeOptions.Datastore != "" {
 				klog.V(4).Infof("Specified zone : %s, datastore : %s", volumeOptions.Zone, volumeOptions.Datastore)
-				dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone)
+				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone)
 				if err != nil {
 					return "", err
 				}
@@ -1198,8 +1205,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 				// Validate if the datastore provided belongs to the zone. If not, fail the operation.
 				found := false
 				for _, ds := range dsList {
-					if ds.Info.Name == volumeOptions.Datastore {
+					if datastoreInfo, found = candidateDatastores[ds.Info.Url]; found {
-						found = true
 						break
 					}
 				}
@@ -1221,19 +1227,14 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 				cleanUpRoutineInitialized = true
 			}
 			cleanUpRoutineInitLock.Unlock()
-			vmOptions, err = vs.setVMOptions(ctx, dc, vs.cfg.Workspace.ResourcePoolPath)
-			if err != nil {
-				klog.Errorf("Failed to set VM options requires to create a vsphere volume. err: %+v", err)
-				return "", err
-			}
 		}
 		if volumeOptions.StoragePolicyName != "" && volumeOptions.Datastore == "" {
 			if len(volumeOptions.Zone) == 0 {
 				klog.V(4).Infof("Selecting a shared datastore as per the storage policy %s", volumeOptions.StoragePolicyName)
-				datastore, err = getPbmCompatibleDatastore(ctx, dc, volumeOptions.StoragePolicyName, vs.nodeManager)
+				datastoreInfo, err = getPbmCompatibleDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, vs.nodeManager)
 			} else {
 				// If zone is specified, first get the datastores in the zone.
-				dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone)
+				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone)
 
 				if err != nil {
 					klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err)
@@ -1249,18 +1250,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 				klog.V(4).Infof("Specified zone : %s. Picking a datastore as per the storage policy %s among the zoned datastores : %s", volumeOptions.Zone,
 					volumeOptions.StoragePolicyName, dsList)
 				// Among the compatible datastores, select the one based on the maximum free space.
-				datastore, err = getPbmCompatibleZonedDatastore(ctx, dc, volumeOptions.StoragePolicyName, dsList)
+				datastoreInfo, err = getPbmCompatibleZonedDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, dsList)
 			}
-			klog.V(1).Infof("Datastore selected as per policy : %s", datastore)
 			if err != nil {
 				klog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err)
 				return "", err
 			}
+			klog.V(1).Infof("Datastore selected as per policy : %s", datastoreInfo.Info.Name)
 		} else {
 			// If zone is specified, pick the datastore in the zone with maximum free space within the zone.
 			if volumeOptions.Datastore == "" && len(volumeOptions.Zone) != 0 {
 				klog.V(4).Infof("Specified zone : %s", volumeOptions.Zone)
-				dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone)
+				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone)
 
 				if err != nil {
 					klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err)
@@ -1273,40 +1274,40 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 					return "", err
 				}
 
-				datastore, err = getMostFreeDatastoreName(ctx, nil, dsList)
+				datastoreInfo, err = getMostFreeDatastore(ctx, nil, dsList)
 				if err != nil {
 					klog.Errorf("Failed to get shared datastore: %+v", err)
 					return "", err
 				}
-				klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.StoragePolicyName, datastore)
+				klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.Zone, datastoreInfo.Info.Name)
 			} else {
 				var sharedDsList []*vclib.DatastoreInfo
 				var err error
 				if len(volumeOptions.Zone) == 0 {
 					// If zone is not provided, get the shared datastore across all node VMs.
-					klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastore)
+					klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastoreName)
-					sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, dc, vs.nodeManager)
+					sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, vs.nodeManager)
 					if err != nil {
 						klog.Errorf("Failed to get shared datastore: %+v", err)
 						return "", err
 					}
 					// Prepare error msg to be used later, if required.
-					err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastore)
+					err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName)
 				} else {
 					// If zone is provided, get the shared datastores in that zone.
-					klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastore, volumeOptions.Zone)
+					klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastoreName, volumeOptions.Zone)
-					sharedDsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone)
+					sharedDsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone)
 					if err != nil {
 						klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err)
 						return "", err
 					}
 					// Prepare error msg to be used later, if required.
-					err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastore, volumeOptions.Zone)
+					err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, volumeOptions.Zone)
 				}
 				found := false
 				// Check if the selected datastore belongs to the list of shared datastores computed.
 				for _, sharedDs := range sharedDsList {
-					if datastore == sharedDs.Info.Name {
+					if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found {
 						klog.V(4).Infof("Datastore validation succeeded")
 						found = true
 						break
@@ -1318,11 +1319,19 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 				}
 			}
 		}
-		ds, err := dc.GetDatastoreByName(ctx, datastore)
+
+		// if datastoreInfo is still not determined, it is an error condition
+		if datastoreInfo == nil {
+			klog.Errorf("Ambigous datastore name %s, cannot be found among: %v", datastoreName, candidateDatastoreInfos)
+			return "", fmt.Errorf("Ambigous datastore name %s", datastoreName)
+		}
+		ds := datastoreInfo.Datastore
+		volumeOptions.Datastore = datastoreInfo.Info.Name
+		vmOptions, err = vs.setVMOptions(ctx, vsi.conn, ds)
 		if err != nil {
+			klog.Errorf("Failed to set VM options required to create a vsphere volume. err: %+v", err)
 			return "", err
 		}
-		volumeOptions.Datastore = datastore
 		kubeVolsPath := filepath.Clean(ds.Path(VolDir)) + "/"
 		err = ds.CreateDirectory(ctx, kubeVolsPath, false)
 		if err != nil && err != vclib.ErrFileAlreadyExist {
@@ -1337,18 +1346,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo
 		}
 		volumePath, err = disk.Create(ctx, ds)
 		if err != nil {
-			klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, datastore, err)
+			klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, ds, err)
 			return "", err
 		}
 		// Get the canonical path for the volume path.
-		canonicalVolumePath, err = getcanonicalVolumePath(ctx, dc, volumePath)
+		canonicalVolumePath, err = getcanonicalVolumePath(ctx, datastoreInfo.Datacenter, volumePath)
 		if err != nil {
-			klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, datastore, err)
+			klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, ds, err)
 			return "", err
 		}
-		if filepath.Base(datastore) != datastore {
+		if filepath.Base(datastoreName) != datastoreName {
 			// If datastore is within cluster, add cluster path to the volumePath
-			canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastore), datastore, 1)
+			canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastoreName), datastoreName, 1)
 		}
 		return canonicalVolumePath, nil
 	}
@@ -1577,12 +1586,29 @@ func (vs *VSphere) GetVolumeLabels(volumePath string) (map[string]string, error)
 		return nil, nil
 	}
 
+	// Find the datastore on which this volume resides
 	datastorePathObj, err := vclib.GetDatastorePathObjFromVMDiskPath(volumePath)
 	if err != nil {
 		klog.Errorf("Failed to get datastore for volume: %v: %+v", volumePath, err)
 		return nil, err
 	}
-	dsZones, err := vs.GetZonesForDatastore(ctx, datastorePathObj.Datastore)
+	dsInfos, err := vs.FindDatastoreByName(ctx, datastorePathObj.Datastore)
+	if err != nil {
+		klog.Errorf("Failed to get datastore by name: %v: %+v", datastorePathObj.Datastore, err)
+		return nil, err
+	}
+	var datastore *vclib.Datastore
+	for _, dsInfo := range dsInfos {
+		if dsInfo.Datastore.Exists(ctx, datastorePathObj.Path) {
+			datastore = dsInfo.Datastore
+		}
+	}
+	if datastore == nil {
+		klog.Errorf("Could not find %s among %v", volumePath, dsInfos)
+		return nil, fmt.Errorf("Could not find the datastore for volume: %s", volumePath)
+	}
+
+	dsZones, err := vs.GetZonesForDatastore(ctx, datastore)
 	if err != nil {
 		klog.Errorf("Failed to get zones for datastore %v: %+v", datastorePathObj.Datastore, err)
 		return nil, err
@@ -1620,25 +1646,16 @@ func (vs *VSphere) collapseZonesInRegion(ctx context.Context, zones []cloudprovi
 }
 
 // GetZonesForDatastore returns all the zones from which this datastore is visible
-func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore string) ([]cloudprovider.Zone, error) {
+func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore *vclib.Datastore) ([]cloudprovider.Zone, error) {
 	vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
 	if err != nil {
 		klog.Errorf("Failed to get vSphere instance: %+v", err)
 		return nil, err
 	}
-	dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
+
-	if err != nil {
-		klog.Errorf("Failed to get datacenter: %+v", err)
-		return nil, err
-	}
 	// get the hosts mounted on this datastore
 	// datastore -> ["host-1", "host-2", "host-3", ...]
-	ds, err := dc.GetDatastoreByName(ctx, datastore)
+	dsHosts, err := datastore.GetDatastoreHostMounts(ctx)
-	if err != nil {
-		klog.Errorf("Failed to get datastore by name: %v: %+v", datastore, err)
-		return nil, err
-	}
-	dsHosts, err := ds.GetDatastoreHostMounts(ctx)
 	if err != nil {
 		klog.Errorf("Failed to get datastore host mounts for %v: %+v", datastore, err)
 		return nil, err

staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere_util.go

@@ -20,20 +20,22 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/vmware/govmomi/find"
 	"github.com/vmware/govmomi/object"
 	"github.com/vmware/govmomi/property"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
 	"github.com/vmware/govmomi/vim25/types"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 
-	"k8s.io/api/core/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/legacy-cloud-providers/vsphere/vclib"
@@ -43,6 +45,7 @@ import (
 const (
 	DatastoreProperty     = "datastore"
 	DatastoreInfoProperty = "info"
+	DatastoreNameProperty = "name"
 	Folder                = "Folder"
 	VirtualMachine        = "VirtualMachine"
 	DummyDiskName         = "kube-dummyDisk.vmdk"
@@ -121,7 +124,7 @@ func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nod
 }
 
 // Get all datastores accessible for the virtual machine object.
-func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) {
+func getSharedDatastoresInK8SCluster(ctx context.Context, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) {
 	nodeVmDetails, err := nodeManager.GetNodeDetails()
 	if err != nil {
 		klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err)
@@ -155,12 +158,6 @@ func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter,
 		}
 	}
 	klog.V(9).Infof("sharedDatastores : %+v", sharedDatastores)
-	sharedDatastores, err = getDatastoresForEndpointVC(ctx, dc, sharedDatastores)
-	if err != nil {
-		klog.Errorf("Failed to get shared datastores from endpoint VC. err: %+v", err)
-		return nil, err
-	}
-	klog.V(9).Infof("sharedDatastores at endpoint VC: %+v", sharedDatastores)
 	return sharedDatastores, nil
 }
 
@@ -182,7 +179,7 @@ func intersect(list1 []*vclib.DatastoreInfo, list2 []*vclib.DatastoreInfo) []*vc
 }
 
 // getMostFreeDatastore gets the best fit compatible datastore by free space.
-func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (string, error) {
+func getMostFreeDatastore(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) {
 	var curMax int64
 	curMax = -1
 	var index int
@@ -193,65 +190,46 @@ func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoL
 			index = i
 		}
 	}
-	return dsInfoList[index].Info.GetDatastoreInfo().Name, nil
+	return dsInfoList[index], nil
 }
 
-// Returns the datastores in the given datacenter by performing lookup based on datastore URL.
+func getPbmCompatibleDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, nodeManager *NodeManager) (*vclib.DatastoreInfo, error) {
-func getDatastoresForEndpointVC(ctx context.Context, dc *vclib.Datacenter, sharedDsInfos []*vclib.DatastoreInfo) ([]*vclib.DatastoreInfo, error) {
+	pbmClient, err := vclib.NewPbmClient(ctx, vcClient)
-	var datastores []*vclib.DatastoreInfo
-	allDsInfoMap, err := dc.GetAllDatastores(ctx)
 	if err != nil {
 		return nil, err
 	}
-	for _, sharedDsInfo := range sharedDsInfos {
-		dsInfo, ok := allDsInfoMap[sharedDsInfo.Info.Url]
-		if ok {
-			datastores = append(datastores, dsInfo)
-		} else {
-			klog.V(4).Infof("Warning: Shared datastore with URL %s does not exist in endpoint VC", sharedDsInfo.Info.Url)
-		}
-	}
-	klog.V(9).Infof("Datastore from endpoint VC: %+v", datastores)
-	return datastores, nil
-}
-
-func getPbmCompatibleDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, nodeManager *NodeManager) (string, error) {
-	pbmClient, err := vclib.NewPbmClient(ctx, dc.Client())
-	if err != nil {
-		return "", err
-	}
 	storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName)
 	if err != nil {
 		klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err)
-		return "", err
+		return nil, err
 	}
-	sharedDs, err := getSharedDatastoresInK8SCluster(ctx, dc, nodeManager)
+	sharedDs, err := getSharedDatastoresInK8SCluster(ctx, nodeManager)
 	if err != nil {
 		klog.Errorf("Failed to get shared datastores. err: %+v", err)
-		return "", err
+		return nil, err
 	}
 	if len(sharedDs) == 0 {
 		msg := "No shared datastores found in the endpoint virtual center"
 		klog.Errorf(msg)
-		return "", errors.New(msg)
+		return nil, errors.New(msg)
 	}
-	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, sharedDs)
+	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, sharedDs)
 	if err != nil {
 		klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v",
 			sharedDs, storagePolicyID, err)
-		return "", err
+		return nil, err
 	}
 	klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores)
-	datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores)
+	datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores)
 	if err != nil {
 		klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err)
-		return "", err
+		return nil, err
 	}
-	klog.V(4).Infof("Most free datastore : %+s", datastore)
+	klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name)
 	return datastore, err
 }
 
-func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) {
+func getDatastoresForZone(ctx context.Context, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) {
 
 	var sharedDatastores []*vclib.DatastoreInfo
 
@@ -279,7 +257,7 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager
 
 			var dsMoList []mo.Datastore
 			pc := property.DefaultCollector(host.Client())
-			properties := []string{DatastoreInfoProperty}
+			properties := []string{DatastoreInfoProperty, DatastoreNameProperty}
 			err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList)
 			if err != nil {
 				klog.Errorf("Failed to get Datastore managed objects from datastore objects."+
@@ -288,12 +266,37 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager
 			}
 			klog.V(9).Infof("Datastore mo details: %+v", dsMoList)
 
+			// find the Datacenter parent for this host
+			mes, err := mo.Ancestors(ctx, host.Client(), pc.Reference(), host.Reference())
+			if err != nil {
+				return nil, err
+			}
+			var dcMoref *types.ManagedObjectReference
+			for i := len(mes) - 1; i > 0; i-- {
+				if mes[i].Self.Type == "Datacenter" {
+					dcMoref = &mes[i].Self
+					break
+				}
+			}
+			if dcMoref == nil {
+				return nil, fmt.Errorf("Failed to find the Datacenter of host %s", host)
+			}
+
+			dc := object.NewDatacenter(host.Client(), *dcMoref)
+			finder := find.NewFinder(host.Client(), false)
+			finder.SetDatacenter(dc)
 			var dsObjList []*vclib.DatastoreInfo
 			for _, dsMo := range dsMoList {
+				// use the finder so that InventoryPath is set correctly in dsObj
+				dsObj, err := finder.Datastore(ctx, dsMo.Name)
+				if err != nil {
+					klog.Errorf("Failed to find datastore named %s in datacenter %s", dsMo.Name, dc)
+					return nil, err
+				}
 				dsObjList = append(dsObjList,
 					&vclib.DatastoreInfo{
-						Datastore: &vclib.Datastore{Datastore: object.NewDatastore(host.Client(), dsMo.Reference()),
+						Datastore: &vclib.Datastore{Datastore: dsObj,
-							Datacenter: nil},
+							Datacenter: &vclib.Datacenter{Datacenter: dc}},
 						Info: dsMo.Info.GetDatastoreInfo()})
 			}
 
@@ -317,40 +320,48 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager
 	return sharedDatastores, nil
 }
 
-func getPbmCompatibleZonedDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (string, error) {
+func getPbmCompatibleZonedDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) {
-	pbmClient, err := vclib.NewPbmClient(ctx, dc.Client())
+	pbmClient, err := vclib.NewPbmClient(ctx, vcClient)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName)
 	if err != nil {
 		klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err)
-		return "", err
+		return nil, err
 	}
-	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, zonedDatastores)
+	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, zonedDatastores)
 	if err != nil {
 		klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v",
 			zonedDatastores, storagePolicyID, err)
-		return "", err
+		return nil, err
 	}
 	klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores)
-	datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores)
+	datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores)
 	if err != nil {
 		klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err)
-		return "", err
+		return nil, err
 	}
-	klog.V(4).Infof("Most free datastore : %+s", datastore)
+	klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name)
 	return datastore, err
 }
 
-func (vs *VSphere) setVMOptions(ctx context.Context, dc *vclib.Datacenter, resourcePoolPath string) (*vclib.VMOptions, error) {
+func (vs *VSphere) setVMOptions(ctx context.Context, connection *vclib.VSphereConnection, ds *vclib.Datastore) (*vclib.VMOptions, error) {
 	var vmOptions vclib.VMOptions
-	resourcePool, err := dc.GetResourcePool(ctx, resourcePoolPath)
+	dsHosts, err := ds.GetDatastoreHostMounts(ctx)
 	if err != nil {
+		klog.Errorf("Failed to get datastore host mounts for %v: %+v", ds, err)
 		return nil, err
 	}
-	klog.V(9).Infof("Resource pool path %s, resourcePool %+v", resourcePoolPath, resourcePool)
+	// pick a host at random to use for Volume creation
-	folder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
+	dsHostMoref := dsHosts[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(dsHosts))]
+	dummyVMHost := object.NewHostSystem(connection.Client, dsHostMoref)
+	resourcePool, err := dummyVMHost.ResourcePool(ctx)
+	if err != nil {
+		klog.Errorf("Failed to get resource pool from host %v", dummyVMHost)
+		return nil, err
+	}
+	folder, err := ds.Datacenter.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
 	if err != nil {
 		return nil, err
 	}
@@ -366,16 +377,13 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) {
 	defer cancel()
 	for {
 		time.Sleep(CleanUpDummyVMRoutineInterval * time.Minute)
-		vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
+		datacenters, err := vs.GetWorkspaceDatacenters(ctx)
 		if err != nil {
-			klog.V(4).Infof("Failed to get VSphere instance with err: %+v. Retrying again...", err)
+			klog.V(4).Infof("Failed to get datacenters from VC. err: %+v", err)
-			continue
-		}
-		dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
-		if err != nil {
-			klog.V(4).Infof("Failed to get the datacenter: %s from VC. err: %+v", vs.cfg.Workspace.Datacenter, err)
 			continue
 		}
+		// Clean up dummy VMs in each datacenter
+		for _, dc := range datacenters {
 			// Get the folder reference for global working directory where the dummy VM needs to be created.
 			vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
 			if err != nil {
@@ -386,13 +394,14 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) {
 			cleanUpDummyVMs := func() {
 				cleanUpDummyVMLock.Lock()
 				defer cleanUpDummyVMLock.Unlock()
-			err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder, dc)
+				err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder)
 				if err != nil {
 					klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err)
 				}
 			}
 			cleanUpDummyVMs()
 		}
+	}
 }
 
 // Get canonical volume path for volume Path.
@@ -586,43 +595,6 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN
 	return nodesToRetry, nil
 }
 
-func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) {
-	isDummyVMPresent := false
-
-	// Create context
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
-	if err != nil {
-		return isDummyVMPresent, err
-	}
-
-	dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
-	if err != nil {
-		return isDummyVMPresent, err
-	}
-
-	vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
-	if err != nil {
-		return isDummyVMPresent, err
-	}
-
-	vms, err := vmFolder.GetVirtualMachines(ctx)
-	if err != nil {
-		return isDummyVMPresent, err
-	}
-
-	for _, vm := range vms {
-		if vm.Name() == vmName {
-			isDummyVMPresent = true
-			break
-		}
-	}
-
-	return isDummyVMPresent, nil
-}
-
 func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) {
 	var nodeName string
 	nodes, err := vs.nodeManager.GetNodeDetails()
@@ -698,3 +670,63 @@ func GetVMUUID() (string, error) {
 	uuid = fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:32])
 	return uuid, nil
 }
+
+// GetWorkspaceDatacenters returns the Datacenter objects that VCP has access to.
+// User can configure the list of datacenters in vsphere.conf. Otherwise all the
+// Datacenters in the configured list of VCs are returned.
+func (vs *VSphere) GetWorkspaceDatacenters(ctx context.Context) ([]*vclib.Datacenter, error) {
+	var datacenterObjs []*vclib.Datacenter
+	for vc, vsi := range vs.vsphereInstanceMap {
+		// ensure connection to VC
+		err := vs.nodeManager.vcConnect(ctx, vsi)
+		if err != nil {
+			return nil, err
+		}
+		if vsi.cfg.Datacenters == "" {
+			vcDatacenterObjs, err := vclib.GetAllDatacenter(ctx, vsi.conn)
+			if err != nil {
+				klog.Errorf("Error fetching list of datacenters from VC %s: %+v", vc, err)
+				return nil, err
+			}
+			datacenterObjs = append(datacenterObjs, vcDatacenterObjs...)
+		} else {
+			datacenters := strings.Split(vsi.cfg.Datacenters, ",")
+			for _, dc := range datacenters {
+				dc = strings.TrimSpace(dc)
+				if dc == "" {
+					continue
+				}
+				datacenterObj, err := vclib.GetDatacenter(ctx, vsi.conn, dc)
+				if err != nil {
+					klog.Errorf("Error fetching datacenter %s from VC %s: %+v", dc, vc, err)
+					return nil, err
+				}
+				datacenterObjs = append(datacenterObjs, datacenterObj)
+			}
+		}
+	}
+	return datacenterObjs, nil
+}
+
+// FindDatastoreByName looks for the given datastore by name across all available datacenters.
+// If more than one Datacenter has a Datastore with the given name, then returns reference to all of them.
+func (vs *VSphere) FindDatastoreByName(ctx context.Context, datastoreName string) ([]*vclib.DatastoreInfo, error) {
+	datacenters, err := vs.GetWorkspaceDatacenters(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var datastoreInfos []*vclib.DatastoreInfo
+	for _, dc := range datacenters {
+		datastoreInfo, err := dc.GetDatastoreInfoByName(ctx, datastoreName)
+		if err != nil {
+			klog.V(9).Infof("Did not find datastore %s in datacenter %s, still looking.", datastoreName, dc.Name())
+			continue
+		}
+		datastoreInfos = append(datastoreInfos, datastoreInfo)
+	}
+	if len(datastoreInfos) == 0 {
+		return nil, fmt.Errorf("Datastore '%s' not found", datastoreName)
+	}
+	klog.V(4).Infof("Found datastore infos %v for datastore %s", datastoreInfos, datastoreName)
+	return datastoreInfos, nil
+}

test/e2e/storage/vsphere/vsphere_volume_datastore.go

@@ -70,7 +70,7 @@ var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]",
 		scParameters[DiskFormat] = ThinDisk
 		err := invokeInvalidDatastoreTestNeg(client, namespace, scParameters)
 		Expect(err).To(HaveOccurred())
-		errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": The specified datastore ` + InvalidDatastore + ` is not a shared datastore across node VMs`
+		errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": Datastore ` + InvalidDatastore + ` not found`
 		if !strings.Contains(err.Error(), errorMsg) {
 			framework.ExpectNoError(err, errorMsg)
 		}

test/e2e/storage/vsphere/vsphere_zone_support.go

@@ -176,6 +176,13 @@ var _ = utils.SIGDescribe("Zone Support", func() {
 		verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones)
 	})
 
+	It("Verify a pod is created on a non-Workspace zone and attached to a dynamically created PV, based on the allowed zones and storage policy specified in storage class", func() {
+		By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneB, compatPolicy))
+		scParameters[SpbmStoragePolicy] = compatPolicy
+		zones = append(zones, zoneB)
+		verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones)
+	})
+
 	It("Verify PVC creation with incompatible storagePolicy and zone combination specified in storage class fails", func() {
 		By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneA, nonCompatPolicy))
 		scParameters[SpbmStoragePolicy] = nonCompatPolicy