Merge pull request #18607 from pmorie/devel/provisioner-cinder

Dynamic provisioner for Cinder

cmd/kube-controller-manager/app/plugins.go

@@ -29,6 +29,7 @@ import (
 	// Volume plugins
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	"k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/aws_ebs"
@@ -92,8 +93,8 @@ func NewVolumeProvisioner(cloud cloudprovider.Interface, flags VolumeConfigFlags
 		return getProvisionablePluginFromVolumePlugins(aws_ebs.ProbeVolumePlugins())
 		//	case cloud != nil && gce.ProviderName == cloud.ProviderName():
 		//		return getProvisionablePluginFromVolumePlugins(gce_pd.ProbeVolumePlugins())
-		//	case cloud != nil && openstack.ProviderName == cloud.ProviderName():
-		//		return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins())
+	case cloud != nil && openstack.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins())
 	}
 	return nil, nil
 }

pkg/cloudprovider/providers/openstack/openstack.go

@@ -1023,3 +1023,41 @@ func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) {
 	}
 	return volume, err
 }
+
+// Create a volume of given size (in GiB)
+func (os *OpenStack) CreateVolume(size int) (volumeName string, err error) {
+
+	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+
+	if err != nil || sClient == nil {
+		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
+		return "", err
+	}
+
+	opts := volumes.CreateOpts{Size: size}
+	vol, err := volumes.Create(sClient, opts).Extract()
+	if err != nil {
+		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
+		return "", err
+	}
+	glog.Infof("Created volume %v", vol.ID)
+	return vol.ID, err
+}
+
+func (os *OpenStack) DeleteVolume(volumeName string) error {
+	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+
+	if err != nil || sClient == nil {
+		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
+		return err
+	}
+	err = volumes.Delete(sClient, volumeName).ExtractErr()
+	if err != nil {
+		glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
+	}
+	return err
+}

pkg/cloudprovider/providers/openstack/openstack_test.go

@@ -198,3 +198,26 @@ func TestZones(t *testing.T) {
 		t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
 	}
 }
+
+func TestVolumes(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	os, err := newOpenStack(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
+	}
+
+	vol, err := os.CreateVolume(1)
+	if err != nil {
+		t.Fatalf("Cannot create a new Cinder volume: %v", err)
+	}
+
+	err = os.DeleteVolume(vol)
+	if err != nil {
+		t.Fatalf("Cannot delete Cinder volume %s: %v", vol, err)
+	}
+
+}

pkg/volume/cinder/cinder.go

@@ -17,11 +17,15 @@ limitations under the License.
 package cinder
 
 import (
+	"errors"
+	"fmt"
 	"os"
 	"path"
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util"
 	"k8s.io/kubernetes/pkg/util/exec"
@@ -39,6 +43,9 @@ type cinderPlugin struct {
 }
 
 var _ volume.VolumePlugin = &cinderPlugin{}
+var _ volume.PersistentVolumePlugin = &cinderPlugin{}
+var _ volume.DeletableVolumePlugin = &cinderPlugin{}
+var _ volume.ProvisionableVolumePlugin = &cinderPlugin{}
 
 const (
 	cinderVolumePluginName = "kubernetes.io/cinder"
@@ -107,12 +114,64 @@ func (plugin *cinderPlugin) newCleanerInternal(volName string, podUID types.UID,
 		}}, nil
 }
 
+func (plugin *cinderPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
+	return plugin.newDeleterInternal(spec, &CinderDiskUtil{})
+}
+
+func (plugin *cinderPlugin) newDeleterInternal(spec *volume.Spec, manager cdManager) (volume.Deleter, error) {
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Cinder == nil {
+		return nil, fmt.Errorf("spec.PersistentVolumeSource.Cinder is nil")
+	}
+	return &cinderVolumeDeleter{
+		&cinderVolume{
+			volName: spec.Name(),
+			pdName:  spec.PersistentVolume.Spec.Cinder.VolumeID,
+			manager: manager,
+			plugin:  plugin,
+		}}, nil
+}
+
+func (plugin *cinderPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
+	if len(options.AccessModes) == 0 {
+		options.AccessModes = plugin.GetAccessModes()
+	}
+	return plugin.newProvisionerInternal(options, &CinderDiskUtil{})
+}
+
+func (plugin *cinderPlugin) newProvisionerInternal(options volume.VolumeOptions, manager cdManager) (volume.Provisioner, error) {
+	return &cinderVolumeProvisioner{
+		cinderVolume: &cinderVolume{
+			manager: manager,
+			plugin:  plugin,
+		},
+		options: options,
+	}, nil
+}
+
+func (plugin *cinderPlugin) getCloudProvider() (*openstack.OpenStack, error) {
+	cloud := plugin.host.GetCloudProvider()
+	if cloud == nil {
+		glog.Errorf("Cloud provider not initialized properly")
+		return nil, errors.New("Cloud provider not initialized properly")
+	}
+
+	os := cloud.(*openstack.OpenStack)
+	if os == nil {
+		return nil, errors.New("Invalid cloud provider: expected OpenStack")
+	}
+	return os, nil
+}
+
 // Abstract interface to PD operations.
 type cdManager interface {
 	// Attaches the disk to the kubelet's host machine.
 	AttachDisk(builder *cinderVolumeBuilder, globalPDPath string) error
 	// Detaches the disk from the kubelet's host machine.
 	DetachDisk(cleaner *cinderVolumeCleaner) error
+	// Creates a volume
+	CreateVolume(provisioner *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error)
+	// Deletes a volume
+	DeleteVolume(deleter *cinderVolumeDeleter) error
 }
 
 var _ volume.Builder = &cinderVolumeBuilder{}
@@ -285,3 +344,66 @@ func (c *cinderVolumeCleaner) TearDownAt(dir string) error {
 	}
 	return nil
 }
+
+type cinderVolumeDeleter struct {
+	*cinderVolume
+}
+
+var _ volume.Deleter = &cinderVolumeDeleter{}
+
+func (r *cinderVolumeDeleter) GetPath() string {
+	name := cinderVolumePluginName
+	return r.plugin.host.GetPodVolumeDir(r.podUID, util.EscapeQualifiedNameForDisk(name), r.volName)
+}
+
+func (r *cinderVolumeDeleter) Delete() error {
+	return r.manager.DeleteVolume(r)
+}
+
+type cinderVolumeProvisioner struct {
+	*cinderVolume
+	options volume.VolumeOptions
+}
+
+var _ volume.Provisioner = &cinderVolumeProvisioner{}
+
+func (c *cinderVolumeProvisioner) Provision(pv *api.PersistentVolume) error {
+	volumeID, sizeGB, err := c.manager.CreateVolume(c)
+	if err != nil {
+		return err
+	}
+	pv.Spec.PersistentVolumeSource.Cinder.VolumeID = volumeID
+	pv.Spec.Capacity = api.ResourceList{
+		api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
+	}
+	return nil
+}
+
+func (c *cinderVolumeProvisioner) NewPersistentVolumeTemplate() (*api.PersistentVolume, error) {
+	// Provide dummy api.PersistentVolume.Spec, it will be filled in
+	// cinderVolumeProvisioner.Provision()
+	return &api.PersistentVolume{
+		ObjectMeta: api.ObjectMeta{
+			GenerateName: "pv-cinder-",
+			Labels:       map[string]string{},
+			Annotations: map[string]string{
+				"kubernetes.io/createdby": "cinder-dynamic-provisioner",
+			},
+		},
+		Spec: api.PersistentVolumeSpec{
+			PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy,
+			AccessModes:                   c.options.AccessModes,
+			Capacity: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): c.options.Capacity,
+			},
+			PersistentVolumeSource: api.PersistentVolumeSource{
+				Cinder: &api.CinderVolumeSource{
+					VolumeID: "dummy",
+					FSType:   "ext4",
+					ReadOnly: false,
+				},
+			},
+		},
+	}, nil
+
+}

pkg/volume/cinder/cinder_test.go

@@ -17,12 +17,14 @@ limitations under the License.
 package cinder
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"testing"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
@@ -73,6 +75,17 @@ func (fake *fakePDManager) DetachDisk(c *cinderVolumeCleaner) error {
 	return nil
 }
 
+func (fake *fakePDManager) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) {
+	return "test-volume-name", 1, nil
+}
+
+func (fake *fakePDManager) DeleteVolume(cd *cinderVolumeDeleter) error {
+	if cd.pdName != "test-volume-name" {
+		return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdName)
+	}
+	return nil
+}
+
 func TestPlugin(t *testing.T) {
 	tmpDir, err := ioutil.TempDir(os.TempDir(), "cinderTest")
 	if err != nil {
@@ -142,4 +155,45 @@ func TestPlugin(t *testing.T) {
 	} else if !os.IsNotExist(err) {
 		t.Errorf("SetUp() failed: %v", err)
 	}
+
+	// Test Provisioner
+	cap := resource.MustParse("100Mi")
+	options := volume.VolumeOptions{
+		Capacity: cap,
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+		},
+		PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete,
+	}
+	provisioner, err := plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
+	persistentSpec, err := provisioner.NewPersistentVolumeTemplate()
+	if err != nil {
+		t.Errorf("NewPersistentVolumeTemplate() failed: %v", err)
+	}
+
+	// get 2nd Provisioner - persistent volume controller will do the same
+	provisioner, err = plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
+	err = provisioner.Provision(persistentSpec)
+	if err != nil {
+		t.Errorf("Provision() failed: %v", err)
+	}
+
+	if persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID != "test-volume-name" {
+		t.Errorf("Provision() returned unexpected volume ID: %s", persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID)
+	}
+	cap = persistentSpec.Spec.Capacity[api.ResourceStorage]
+	size := cap.Value()
+	if size != 1024*1024*1024 {
+		t.Errorf("Provision() returned unexpected volume size: %v", size)
+	}
+
+	// Test Deleter
+	volSpec := &volume.Spec{
+		PersistentVolume: persistentSpec,
+	}
+	deleter, err := plug.(*cinderPlugin).newDeleterInternal(volSpec, &fakePDManager{})
+	err = deleter.Delete()
+	if err != nil {
+		t.Errorf("Deleter() failed: %v", err)
+	}
 }

pkg/volume/cinder/cinder_util.go

@@ -26,9 +26,9 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	"k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume"
 )
 
 type CinderDiskUtil struct{}
@@ -40,12 +40,11 @@ func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeBuilder, globalPDPath stri
 	if b.readOnly {
 		options = append(options, "ro")
 	}
-	cloud := b.plugin.host.GetCloudProvider()
-	if cloud == nil {
-		glog.Errorf("Cloud provider not initialized properly")
-		return errors.New("Cloud provider not initialized properly")
+	cloud, err := b.plugin.getCloudProvider()
+	if err != nil {
+		return err
 	}
-	diskid, err := cloud.(*openstack.OpenStack).AttachDisk(b.pdName)
+	diskid, err := cloud.AttachDisk(b.pdName)
 	if err != nil {
 		return err
 	}
@@ -119,19 +118,50 @@ func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeCleaner) error {
 	}
 	glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath)
 
-	cloud := cd.plugin.host.GetCloudProvider()
-	if cloud == nil {
-		glog.Errorf("Cloud provider not initialized properly")
-		return errors.New("Cloud provider not initialized properly")
+	cloud, err := cd.plugin.getCloudProvider()
+	if err != nil {
+		return err
 	}
 
-	if err := cloud.(*openstack.OpenStack).DetachDisk(cd.pdName); err != nil {
+	if err = cloud.DetachDisk(cd.pdName); err != nil {
 		return err
 	}
 	glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName)
 	return nil
 }
 
+func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error {
+	cloud, err := cd.plugin.getCloudProvider()
+	if err != nil {
+		return err
+	}
+
+	if err = cloud.DeleteVolume(cd.pdName); err != nil {
+		glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err)
+		return err
+	}
+	glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName)
+	return nil
+}
+
+func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) {
+	cloud, err := c.plugin.getCloudProvider()
+	if err != nil {
+		return "", 0, err
+	}
+
+	volSizeBytes := c.options.Capacity.Value()
+	// Cinder works with gigabytes, convert to GiB with rounding up
+	volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024))
+	name, err := cloud.CreateVolume(volSizeGB)
+	if err != nil {
+		glog.V(2).Infof("Error creating cinder volume: %v", err)
+		return "", 0, err
+	}
+	glog.V(2).Infof("Successfully created cinder volume %s", name)
+	return name, volSizeGB, nil
+}
+
 type cinderSafeFormatAndMount struct {
 	mount.Interface
 	runner exec.Interface