Optimize the time taken to create Persistent volumes with VSAN storage capabilities at scale and handle VPXD crashes

This commit is contained in:
Balu Dontu 2017-04-26 13:33:21 -07:00
parent e8abe44980
commit 6228765b43

View File

@ -28,6 +28,7 @@ import (
"runtime"
"strings"
"sync"
"time"
"gopkg.in/gcfg.v1"
@ -51,27 +52,33 @@ import (
)
const (
ProviderName = "vsphere"
ActivePowerState = "poweredOn"
SCSIControllerType = "scsi"
LSILogicControllerType = "lsiLogic"
BusLogicControllerType = "busLogic"
PVSCSIControllerType = "pvscsi"
LSILogicSASControllerType = "lsiLogic-sas"
SCSIControllerLimit = 4
SCSIControllerDeviceLimit = 15
SCSIDeviceSlots = 16
SCSIReservedSlot = 7
ThinDiskType = "thin"
PreallocatedDiskType = "preallocated"
EagerZeroedThickDiskType = "eagerZeroedThick"
ZeroedThickDiskType = "zeroedThick"
VolDir = "kubevols"
RoundTripperDefaultCount = 3
DummyVMName = "kubernetes-helper-vm"
VSANDatastoreType = "vsan"
MAC_OUI_VC = "00:50:56"
MAC_OUI_ESX = "00:0c:29"
ProviderName = "vsphere"
ActivePowerState = "poweredOn"
SCSIControllerType = "scsi"
LSILogicControllerType = "lsiLogic"
BusLogicControllerType = "busLogic"
PVSCSIControllerType = "pvscsi"
LSILogicSASControllerType = "lsiLogic-sas"
SCSIControllerLimit = 4
SCSIControllerDeviceLimit = 15
SCSIDeviceSlots = 16
SCSIReservedSlot = 7
ThinDiskType = "thin"
PreallocatedDiskType = "preallocated"
EagerZeroedThickDiskType = "eagerZeroedThick"
ZeroedThickDiskType = "zeroedThick"
VolDir = "kubevols"
RoundTripperDefaultCount = 3
DummyVMPrefixName = "vsphere-k8s"
VSANDatastoreType = "vsan"
MAC_OUI_VC = "00:50:56"
MAC_OUI_ESX = "00:0c:29"
DiskNotFoundErrMsg = "No vSphere disk ID found"
NoDiskUUIDFoundErrMsg = "No disk UUID found"
NoDevicesFoundErrMsg = "No devices found"
NonSupportedControllerTypeErrMsg = "Disk is attached to non-supported controller type"
FileAlreadyExistErrMsg = "File requested already exist"
CleanUpDummyVMRoutine_Interval = 5
)
// Controller types that are currently supported for hot attach of disks
@ -91,14 +98,17 @@ var diskFormatValidType = map[string]string{
}
var DiskformatValidOptions = generateDiskFormatValidOptions()
var cleanUpRoutineInitialized = false
var ErrNoDiskUUIDFound = errors.New("No disk UUID found")
var ErrNoDiskIDFound = errors.New("No vSphere disk ID found")
var ErrNoDevicesFound = errors.New("No devices found")
var ErrNonSupportedControllerType = errors.New("Disk is attached to non-supported controller type")
var ErrFileAlreadyExist = errors.New("File requested already exist")
var ErrNoDiskUUIDFound = errors.New(NoDiskUUIDFoundErrMsg)
var ErrNoDiskIDFound = errors.New(DiskNotFoundErrMsg)
var ErrNoDevicesFound = errors.New(NoDevicesFoundErrMsg)
var ErrNonSupportedControllerType = errors.New(NonSupportedControllerTypeErrMsg)
var ErrFileAlreadyExist = errors.New(FileAlreadyExistErrMsg)
var clientLock sync.Mutex
var cleanUpRoutineInitLock sync.Mutex
var cleanUpDummyVMLock sync.RWMutex
// VSphere is an implementation of cloud provider Interface for VSphere.
type VSphere struct {
@ -1239,6 +1249,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
// 1. Create dummy VM if not already present.
// 2. Add a new disk to the VM by performing VM reconfigure.
// 3. Detach the new disk from the dummy VM.
// 4. Delete the dummy VM.
if volumeOptions.StorageProfileData != "" {
// Check if the datastore is VSAN if any capability requirements are specified.
// VSphere cloud provider now only supports VSAN capabilities requirements
@ -1253,13 +1264,27 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
" So, please specify a valid VSAN datastore in Storage class definition.", datastore)
}
// Check if the DummyVM exists in kubernetes cluster folder.
// Acquire a read lock to ensure multiple PVC requests can be processed simultaneously.
cleanUpDummyVMLock.RLock()
defer cleanUpDummyVMLock.RUnlock()
// Create a new background routine that will delete any dummy VM's that are left stale.
// This routine will get executed for every 5 minutes and gets initiated only once in its entire lifetime.
cleanUpRoutineInitLock.Lock()
if !cleanUpRoutineInitialized {
go vs.cleanUpDummyVMs(DummyVMPrefixName)
cleanUpRoutineInitialized = true
}
cleanUpRoutineInitLock.Unlock()
// Check if the VM exists in kubernetes cluster folder.
// The kubernetes cluster folder - vs.cfg.Global.WorkingDir is where all the nodes in the kubernetes cluster are created.
vmRegex := vs.cfg.Global.WorkingDir + DummyVMName
dummyVMFullName := DummyVMPrefixName + "-" + volumeOptions.Name
vmRegex := vs.cfg.Global.WorkingDir + dummyVMFullName
dummyVM, err := f.VirtualMachine(ctx, vmRegex)
if err != nil {
// 1. Create dummy VM and return the VM reference.
dummyVM, err = vs.createDummyVM(ctx, dc, ds)
// 1. Create a dummy VM and return the VM reference.
dummyVM, err = vs.createDummyVM(ctx, dc, ds, dummyVMFullName)
if err != nil {
return "", err
}
@ -1267,17 +1292,37 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
// 2. Reconfigure the VM to attach the disk with the VSAN policy configured.
vmDiskPath, err := vs.createVirtualDiskWithPolicy(ctx, dc, ds, dummyVM, volumeOptions)
fileAlreadyExist := false
if err != nil {
glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", DummyVMName, err)
return "", err
vmDiskPath = filepath.Clean(ds.Path(VolDir)) + "/" + volumeOptions.Name + ".vmdk"
errorMessage := fmt.Sprintf("Cannot complete the operation because the file or folder %s already exists", vmDiskPath)
if errorMessage == err.Error() {
//Skip error and continue to detach the disk as the disk was already created on the datastore.
fileAlreadyExist = true
glog.V(1).Infof("File: %v already exists", vmDiskPath)
} else {
glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err)
return "", err
}
}
dummyVMNodeName := vmNameToNodeName(DummyVMName)
dummyVMNodeName := vmNameToNodeName(dummyVMFullName)
// 3. Detach the disk from the dummy VM.
err = vs.DetachDisk(vmDiskPath, dummyVMNodeName)
if err != nil {
glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmDiskPath, DummyVMName, err)
return "", fmt.Errorf("Failed to create the volume: %q with err: %+v", volumeOptions.Name, err)
if DiskNotFoundErrMsg == err.Error() && fileAlreadyExist {
// Skip error if disk was already detached from the dummy VM but still present on the datastore.
glog.V(1).Infof("File: %v is already detached", vmDiskPath)
} else {
glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmDiskPath, dummyVMFullName, err)
return "", fmt.Errorf("Failed to create the volume: %q with err: %+v", volumeOptions.Name, err)
}
}
// 4. Delete the dummy VM
err = deleteVM(ctx, dummyVM)
if err != nil {
return "", fmt.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err)
}
destVolPath = vmDiskPath
} else {
@ -1318,6 +1363,22 @@ func (vs *VSphere) DeleteVolume(vmDiskPath string) error {
if filepath.Ext(vmDiskPath) != ".vmdk" {
vmDiskPath += ".vmdk"
}
// Get the vmDisk Name
diskNameWithExt := path.Base(vmDiskPath)
diskName := strings.TrimSuffix(diskNameWithExt, filepath.Ext(diskNameWithExt))
// Search for the dummyVM if present and delete it.
dummyVMFullName := DummyVMPrefixName + "-" + diskName
vmRegex := vs.cfg.Global.WorkingDir + dummyVMFullName
dummyVM, err := f.VirtualMachine(ctx, vmRegex)
if err == nil {
err = deleteVM(ctx, dummyVM)
if err != nil {
return fmt.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err)
}
}
// Delete virtual disk
task, err := virtualDiskManager.DeleteVirtualDisk(ctx, vmDiskPath, dc)
if err != nil {
@ -1356,26 +1417,144 @@ func (vs *VSphere) NodeExists(nodeName k8stypes.NodeName) (bool, error) {
return false, nil
}
func (vs *VSphere) createDummyVM(ctx context.Context, datacenter *object.Datacenter, datastore *object.Datastore) (*object.VirtualMachine, error) {
// A background routine which will be responsible for deleting stale dummy VM's.
func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) {
// Create context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for {
time.Sleep(CleanUpDummyVMRoutine_Interval * time.Minute)
// Ensure client is logged in and session is valid
err := vSphereLogin(ctx, vs)
if err != nil {
glog.V(4).Infof("[cleanUpDummyVMs] Unable to login to vSphere with err: %+v", err)
continue
}
// Create a new finder
f := find.NewFinder(vs.client.Client, true)
// Fetch and set data center
dc, err := f.Datacenter(ctx, vs.cfg.Global.Datacenter)
if err != nil {
glog.V(4).Infof("[cleanUpDummyVMs] Unable to fetch the datacenter: %q with err: %+v", vs.cfg.Global.Datacenter, err)
continue
}
f.SetDatacenter(dc)
// Get the folder reference for global working directory where the dummy VM needs to be created.
vmFolder, err := getFolder(ctx, vs.client, vs.cfg.Global.Datacenter, vs.cfg.Global.WorkingDir)
if err != nil {
glog.V(4).Infof("[cleanUpDummyVMs] Unable to get the kubernetes folder: %q reference with err: %+v", vs.cfg.Global.WorkingDir, err)
continue
}
// A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests.
cleanUpDummyVMLock.Lock()
dummyVMRefList, err := getDummyVMList(ctx, vs.client, vmFolder, dummyVMPrefix)
if err != nil {
glog.V(4).Infof("[cleanUpDummyVMs] Unable to get dummy VM list in the kubernetes cluster: %q reference with err: %+v", vs.cfg.Global.WorkingDir, err)
cleanUpDummyVMLock.Unlock()
continue
}
for _, dummyVMRef := range dummyVMRefList {
err = deleteVM(ctx, dummyVMRef)
if err != nil {
glog.V(4).Infof("[cleanUpDummyVMs] Unable to delete dummy VM: %q with err: %+v", dummyVMRef.Name(), err)
continue
}
}
cleanUpDummyVMLock.Unlock()
}
}
// Get the dummy VM list from the kubernetes working directory.
func getDummyVMList(ctx context.Context, c *govmomi.Client, vmFolder *object.Folder, dummyVMPrefix string) ([]*object.VirtualMachine, error) {
vmFolders, err := vmFolder.Children(ctx)
if err != nil {
glog.V(4).Infof("Unable to retrieve the virtual machines from the kubernetes cluster: %+v", vmFolder)
return nil, err
}
var dummyVMRefList []*object.VirtualMachine
pc := property.DefaultCollector(c.Client)
for _, vmFolder := range vmFolders {
if vmFolder.Reference().Type == "VirtualMachine" {
var vmRefs []types.ManagedObjectReference
var vmMorefs []mo.VirtualMachine
vmRefs = append(vmRefs, vmFolder.Reference())
err = pc.Retrieve(ctx, vmRefs, []string{"name"}, &vmMorefs)
if err != nil {
return nil, err
}
if strings.HasPrefix(vmMorefs[0].Name, dummyVMPrefix) {
dummyVMRefList = append(dummyVMRefList, object.NewVirtualMachine(c.Client, vmRefs[0]))
}
}
}
return dummyVMRefList, nil
}
func (vs *VSphere) createDummyVM(ctx context.Context, datacenter *object.Datacenter, datastore *object.Datastore, vmName string) (*object.VirtualMachine, error) {
// Create a virtual machine config spec with 1 SCSI adapter.
virtualMachineConfigSpec := types.VirtualMachineConfigSpec{
Name: DummyVMName,
Name: vmName,
Files: &types.VirtualMachineFileInfo{
VmPathName: "[" + datastore.Name() + "]",
},
NumCPUs: 1,
MemoryMB: 4,
DeviceChange: []types.BaseVirtualDeviceConfigSpec{
&types.VirtualDeviceConfigSpec{
Operation: types.VirtualDeviceConfigSpecOperationAdd,
Device: &types.ParaVirtualSCSIController{
VirtualSCSIController: types.VirtualSCSIController{
SharedBus: types.VirtualSCSISharingNoSharing,
VirtualController: types.VirtualController{
BusNumber: 0,
VirtualDevice: types.VirtualDevice{
Key: 1000,
},
},
},
},
},
},
}
// Create a new finder
f := find.NewFinder(vs.client.Client, true)
f.SetDatacenter(datacenter)
// Get the resource pool for current node. This is where dummy VM will be created.
resourcePool, err := vs.getCurrentNodeResourcePool(ctx, datacenter)
if err != nil {
return nil, err
}
// Get the folder reference for global working directory where the dummy VM needs to be created.
vmFolder, err := getFolder(ctx, vs.client, vs.cfg.Global.Datacenter, vs.cfg.Global.WorkingDir)
if err != nil {
return nil, fmt.Errorf("Failed to get the folder reference for %q", vs.cfg.Global.WorkingDir)
return nil, fmt.Errorf("Failed to get the folder reference for %q with err: %+v", vs.cfg.Global.WorkingDir, err)
}
task, err := vmFolder.CreateVM(ctx, virtualMachineConfigSpec, resourcePool, nil)
if err != nil {
return nil, err
}
dummyVMTaskInfo, err := task.WaitForResult(ctx, nil)
if err != nil {
return nil, err
}
vmRef := dummyVMTaskInfo.Result.(object.Reference)
dummyVM := object.NewVirtualMachine(vs.client.Client, vmRef.Reference())
return dummyVM, nil
}
func (vs *VSphere) getCurrentNodeResourcePool(ctx context.Context, datacenter *object.Datacenter) (*object.ResourcePool, error) {
// Create a new finder
f := find.NewFinder(vs.client.Client, true)
f.SetDatacenter(datacenter)
vmRegex := vs.cfg.Global.WorkingDir + vs.localInstanceID
currentVM, err := f.VirtualMachine(ctx, vmRegex)
if err != nil {
@ -1394,18 +1573,7 @@ func (vs *VSphere) createDummyVM(ctx context.Context, datacenter *object.Datacen
return nil, err
}
task, err := vmFolder.CreateVM(ctx, virtualMachineConfigSpec, resourcePool, nil)
if err != nil {
return nil, err
}
dummyVMTaskInfo, err := task.WaitForResult(ctx, nil)
if err != nil {
return nil, err
}
dummyVM := dummyVMTaskInfo.Result.(*object.VirtualMachine)
return dummyVM, nil
return resourcePool, nil
}
// Creates a virtual disk with the policy configured to the disk.
@ -1421,28 +1589,7 @@ func (vs *VSphere) createVirtualDiskWithPolicy(ctx context.Context, datacenter *
var diskControllerType = vs.cfg.Disk.SCSIControllerType
// find SCSI controller of particular type from VM devices
scsiControllersOfRequiredType := getSCSIControllersOfType(vmDevices, diskControllerType)
scsiController := getAvailableSCSIController(scsiControllersOfRequiredType)
var newSCSIController types.BaseVirtualDevice
if scsiController == nil {
newSCSIController, err = createAndAttachSCSIControllerToVM(ctx, virtualMachine, diskControllerType)
if err != nil {
glog.Errorf("Failed to create SCSI controller for VM :%q with err: %+v", virtualMachine.Name(), err)
return "", err
}
// verify scsi controller in virtual machine
vmDevices, err := virtualMachine.Device(ctx)
if err != nil {
return "", err
}
scsiController = getSCSIController(vmDevices, diskControllerType)
if scsiController == nil {
glog.Errorf("cannot find SCSI controller in VM")
// attempt clean up of scsi controller
cleanUpController(ctx, newSCSIController, vmDevices, virtualMachine)
return "", fmt.Errorf("cannot find SCSI controller in VM")
}
}
scsiController := scsiControllersOfRequiredType[0]
kubeVolsPath := filepath.Clean(datastore.Path(VolDir)) + "/"
// Create a kubevols directory in the datastore if one doesn't exist.
@ -1673,3 +1820,12 @@ func getFolder(ctx context.Context, c *govmomi.Client, datacenterName string, fo
return resultFolder, nil
}
// Delete the VM.
func deleteVM(ctx context.Context, vm *object.VirtualMachine) error {
destroyTask, err := vm.Destroy(ctx)
if err != nil {
return err
}
return destroyTask.Wait(ctx)
}