From 2f06ebf9b765eb610f4d808cc784eef2e1c8c6df Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 15 Dec 2015 12:38:59 +0100 Subject: [PATCH 1/2] Implement Creater and Deleter interfaces for Cinder. --- cmd/kube-controller-manager/app/plugins.go | 5 +- .../providers/openstack/openstack.go | 38 +++++++ .../providers/openstack/openstack_test.go | 23 ++++ pkg/volume/cinder/cinder.go | 106 ++++++++++++++++++ pkg/volume/cinder/cinder_test.go | 54 +++++++++ pkg/volume/cinder/cinder_util.go | 35 ++++++ 6 files changed, 259 insertions(+), 2 deletions(-) diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 12a2017fb33..b58fcda5dca 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -29,6 +29,7 @@ import ( // Volume plugins "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/aws_ebs" @@ -92,8 +93,8 @@ func NewVolumeProvisioner(cloud cloudprovider.Interface, flags VolumeConfigFlags return getProvisionablePluginFromVolumePlugins(aws_ebs.ProbeVolumePlugins()) // case cloud != nil && gce.ProviderName == cloud.ProviderName(): // return getProvisionablePluginFromVolumePlugins(gce_pd.ProbeVolumePlugins()) - // case cloud != nil && openstack.ProviderName == cloud.ProviderName(): - // return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins()) + case cloud != nil && openstack.ProviderName == cloud.ProviderName(): + return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins()) } return nil, nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 6dfbe9aae50..bfc34cd5410 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -1023,3 +1023,41 @@ func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) { } return volume, err } + +// Create a volume of given size (in GiB) +func (os *OpenStack) CreateVolume(size int) (volumeName string, err error) { + + sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{ + Region: os.region, + }) + + if err != nil || sClient == nil { + glog.Errorf("Unable to initialize cinder client for region: %s", os.region) + return "", err + } + + opts := volumes.CreateOpts{Size: size} + vol, err := volumes.Create(sClient, opts).Extract() + if err != nil { + glog.Errorf("Failed to create a %d GB volume: %v", size, err) + return "", err + } + glog.Infof("Created volume %v", vol.ID) + return vol.ID, err +} + +func (os *OpenStack) DeleteVolume(volumeName string) error { + sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{ + Region: os.region, + }) + + if err != nil || sClient == nil { + glog.Errorf("Unable to initialize cinder client for region: %s", os.region) + return err + } + err = volumes.Delete(sClient, volumeName).ExtractErr() + if err != nil { + glog.Errorf("Cannot delete volume %s: %v", volumeName, err) + } + return err +} diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go index 0c566767616..be70f496d02 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ b/pkg/cloudprovider/providers/openstack/openstack_test.go @@ -198,3 +198,26 @@ func TestZones(t *testing.T) { t.Fatalf("GetZone() returned wrong region (%s)", zone.Region) } } + +func TestVolumes(t *testing.T) { + cfg, ok := configFromEnv() + if !ok { + t.Skipf("No config found in environment") + } + + os, err := newOpenStack(cfg) + if err != nil { + t.Fatalf("Failed to construct/authenticate OpenStack: %s", err) + } + + vol, err := os.CreateVolume(1) + if err != nil { + t.Fatalf("Cannot create a new Cinder volume: %v", err) + } + + err = os.DeleteVolume(vol) + if err != nil { + t.Fatalf("Cannot delete Cinder volume %s: %v", vol, err) + } + +} diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index f1366a04831..1d62f696a86 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -17,11 +17,13 @@ limitations under the License. package cinder import ( + "fmt" "os" "path" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" @@ -39,6 +41,9 @@ type cinderPlugin struct { } var _ volume.VolumePlugin = &cinderPlugin{} +var _ volume.PersistentVolumePlugin = &cinderPlugin{} +var _ volume.DeletableVolumePlugin = &cinderPlugin{} +var _ volume.ProvisionableVolumePlugin = &cinderPlugin{} const ( cinderVolumePluginName = "kubernetes.io/cinder" @@ -107,12 +112,50 @@ func (plugin *cinderPlugin) newCleanerInternal(volName string, podUID types.UID, }}, nil } +func (plugin *cinderPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + return plugin.newDeleterInternal(spec, &CinderDiskUtil{}) +} + +func (plugin *cinderPlugin) newDeleterInternal(spec *volume.Spec, manager cdManager) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Cinder == nil { + return nil, fmt.Errorf("spec.PersistentVolumeSource.Cinder is nil") + } + return &cinderVolumeDeleter{ + &cinderVolume{ + volName: spec.Name(), + pdName: spec.PersistentVolume.Spec.Cinder.VolumeID, + manager: manager, + plugin: plugin, + }}, nil +} + +func (plugin *cinderPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + if len(options.AccessModes) == 0 { + options.AccessModes = plugin.GetAccessModes() + } + return plugin.newProvisionerInternal(options, &CinderDiskUtil{}) +} + +func (plugin *cinderPlugin) newProvisionerInternal(options volume.VolumeOptions, manager cdManager) (volume.Provisioner, error) { + return &cinderVolumeProvisioner{ + cinderVolume: &cinderVolume{ + manager: manager, + plugin: plugin, + }, + options: options, + }, nil +} + // Abstract interface to PD operations. type cdManager interface { // Attaches the disk to the kubelet's host machine. AttachDisk(builder *cinderVolumeBuilder, globalPDPath string) error // Detaches the disk from the kubelet's host machine. DetachDisk(cleaner *cinderVolumeCleaner) error + // Creates a volume + CreateVolume(provisioner *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) + // Deletes a volume + DeleteVolume(deleter *cinderVolumeDeleter) error } var _ volume.Builder = &cinderVolumeBuilder{} @@ -285,3 +328,66 @@ func (c *cinderVolumeCleaner) TearDownAt(dir string) error { } return nil } + +type cinderVolumeDeleter struct { + *cinderVolume +} + +var _ volume.Deleter = &cinderVolumeDeleter{} + +func (r *cinderVolumeDeleter) GetPath() string { + name := cinderVolumePluginName + return r.plugin.host.GetPodVolumeDir(r.podUID, util.EscapeQualifiedNameForDisk(name), r.volName) +} + +func (r *cinderVolumeDeleter) Delete() error { + return r.manager.DeleteVolume(r) +} + +type cinderVolumeProvisioner struct { + *cinderVolume + options volume.VolumeOptions +} + +var _ volume.Provisioner = &cinderVolumeProvisioner{} + +func (c *cinderVolumeProvisioner) Provision(pv *api.PersistentVolume) error { + volumeID, sizeGB, err := c.manager.CreateVolume(c) + if err != nil { + return err + } + pv.Spec.PersistentVolumeSource.Cinder.VolumeID = volumeID + pv.Spec.Capacity = api.ResourceList{ + api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)), + } + return nil +} + +func (c *cinderVolumeProvisioner) NewPersistentVolumeTemplate() (*api.PersistentVolume, error) { + // Provide dummy api.PersistentVolume.Spec, it will be filled in + // cinderVolumeProvisioner.Provision() + return &api.PersistentVolume{ + ObjectMeta: api.ObjectMeta{ + GenerateName: "pv-cinder-", + Labels: map[string]string{}, + Annotations: map[string]string{ + "kubernetes.io/createdby": "cinder-dynamic-provisioner", + }, + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy, + AccessModes: c.options.AccessModes, + Capacity: api.ResourceList{ + api.ResourceName(api.ResourceStorage): c.options.Capacity, + }, + PersistentVolumeSource: api.PersistentVolumeSource{ + Cinder: &api.CinderVolumeSource{ + VolumeID: "dummy", + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + }, nil + +} diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index c89ad0e55a1..e22519f10d4 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -17,12 +17,14 @@ limitations under the License. package cinder import ( + "fmt" "io/ioutil" "os" "path" "testing" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -73,6 +75,17 @@ func (fake *fakePDManager) DetachDisk(c *cinderVolumeCleaner) error { return nil } +func (fake *fakePDManager) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) { + return "test-volume-name", 1, nil +} + +func (fake *fakePDManager) DeleteVolume(cd *cinderVolumeDeleter) error { + if cd.pdName != "test-volume-name" { + return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdName) + } + return nil +} + func TestPlugin(t *testing.T) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cinderTest") if err != nil { @@ -142,4 +155,45 @@ func TestPlugin(t *testing.T) { } else if !os.IsNotExist(err) { t.Errorf("SetUp() failed: %v", err) } + + // Test Provisioner + cap := resource.MustParse("100Mi") + options := volume.VolumeOptions{ + Capacity: cap, + AccessModes: []api.PersistentVolumeAccessMode{ + api.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete, + } + provisioner, err := plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{}) + persistentSpec, err := provisioner.NewPersistentVolumeTemplate() + if err != nil { + t.Errorf("NewPersistentVolumeTemplate() failed: %v", err) + } + + // get 2nd Provisioner - persistent volume controller will do the same + provisioner, err = plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{}) + err = provisioner.Provision(persistentSpec) + if err != nil { + t.Errorf("Provision() failed: %v", err) + } + + if persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID != "test-volume-name" { + t.Errorf("Provision() returned unexpected volume ID: %s", persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID) + } + cap = persistentSpec.Spec.Capacity[api.ResourceStorage] + size := cap.Value() + if size != 1024*1024*1024 { + t.Errorf("Provision() returned unexpected volume size: %v", size) + } + + // Test Deleter + volSpec := &volume.Spec{ + PersistentVolume: persistentSpec, + } + deleter, err := plug.(*cinderPlugin).newDeleterInternal(volSpec, &fakePDManager{}) + err = deleter.Delete() + if err != nil { + t.Errorf("Deleter() failed: %v", err) + } } diff --git a/pkg/volume/cinder/cinder_util.go b/pkg/volume/cinder/cinder_util.go index 685333a436e..22aef4cbe19 100644 --- a/pkg/volume/cinder/cinder_util.go +++ b/pkg/volume/cinder/cinder_util.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" ) type CinderDiskUtil struct{} @@ -132,6 +133,40 @@ func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeCleaner) error { return nil } +func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error { + cloud := cd.plugin.host.GetCloudProvider() + if cloud == nil { + glog.Errorf("Cloud provider not initialized properly") + return errors.New("Cloud provider not initialized properly") + } + + if err := cloud.(*openstack.OpenStack).DeleteVolume(cd.pdName); err != nil { + glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err) + return err + } + glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName) + return nil +} + +func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) { + cloud := c.plugin.host.GetCloudProvider() + if cloud == nil { + glog.Errorf("Cloud provider not initialized properly") + return "", 0, errors.New("Cloud provider not initialized properly") + } + + volSizeBytes := c.options.Capacity.Value() + // Cinder works with gigabytes, convert to GiB with rounding up + volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) + name, err := cloud.(*openstack.OpenStack).CreateVolume(volSizeGB) + if err != nil { + glog.V(2).Infof("Error creating cinder volume: %v", err) + return "", 0, err + } + glog.V(2).Infof("Successfully created cinder volume %s", name) + return name, volSizeGB, nil +} + type cinderSafeFormatAndMount struct { mount.Interface runner exec.Interface From e8f08a969cedb605a6840c7123d04d4861a46c60 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 14 Dec 2015 14:55:26 +0100 Subject: [PATCH 2/2] Extract common function for OpenStack checking. --- pkg/volume/cinder/cinder.go | 16 ++++++++++++++ pkg/volume/cinder/cinder_util.go | 37 ++++++++++++++------------------ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 1d62f696a86..b3d30d74b2c 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -17,6 +17,7 @@ limitations under the License. package cinder import ( + "errors" "fmt" "os" "path" @@ -24,6 +25,7 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" @@ -146,6 +148,20 @@ func (plugin *cinderPlugin) newProvisionerInternal(options volume.VolumeOptions, }, nil } +func (plugin *cinderPlugin) getCloudProvider() (*openstack.OpenStack, error) { + cloud := plugin.host.GetCloudProvider() + if cloud == nil { + glog.Errorf("Cloud provider not initialized properly") + return nil, errors.New("Cloud provider not initialized properly") + } + + os := cloud.(*openstack.OpenStack) + if os == nil { + return nil, errors.New("Invalid cloud provider: expected OpenStack") + } + return os, nil +} + // Abstract interface to PD operations. type cdManager interface { // Attaches the disk to the kubelet's host machine. diff --git a/pkg/volume/cinder/cinder_util.go b/pkg/volume/cinder/cinder_util.go index 22aef4cbe19..69982e789d9 100644 --- a/pkg/volume/cinder/cinder_util.go +++ b/pkg/volume/cinder/cinder_util.go @@ -26,7 +26,6 @@ import ( "time" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -41,12 +40,11 @@ func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeBuilder, globalPDPath stri if b.readOnly { options = append(options, "ro") } - cloud := b.plugin.host.GetCloudProvider() - if cloud == nil { - glog.Errorf("Cloud provider not initialized properly") - return errors.New("Cloud provider not initialized properly") + cloud, err := b.plugin.getCloudProvider() + if err != nil { + return err } - diskid, err := cloud.(*openstack.OpenStack).AttachDisk(b.pdName) + diskid, err := cloud.AttachDisk(b.pdName) if err != nil { return err } @@ -120,13 +118,12 @@ func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeCleaner) error { } glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath) - cloud := cd.plugin.host.GetCloudProvider() - if cloud == nil { - glog.Errorf("Cloud provider not initialized properly") - return errors.New("Cloud provider not initialized properly") + cloud, err := cd.plugin.getCloudProvider() + if err != nil { + return err } - if err := cloud.(*openstack.OpenStack).DetachDisk(cd.pdName); err != nil { + if err = cloud.DetachDisk(cd.pdName); err != nil { return err } glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName) @@ -134,13 +131,12 @@ func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeCleaner) error { } func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error { - cloud := cd.plugin.host.GetCloudProvider() - if cloud == nil { - glog.Errorf("Cloud provider not initialized properly") - return errors.New("Cloud provider not initialized properly") + cloud, err := cd.plugin.getCloudProvider() + if err != nil { + return err } - if err := cloud.(*openstack.OpenStack).DeleteVolume(cd.pdName); err != nil { + if err = cloud.DeleteVolume(cd.pdName); err != nil { glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err) return err } @@ -149,16 +145,15 @@ func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error { } func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) { - cloud := c.plugin.host.GetCloudProvider() - if cloud == nil { - glog.Errorf("Cloud provider not initialized properly") - return "", 0, errors.New("Cloud provider not initialized properly") + cloud, err := c.plugin.getCloudProvider() + if err != nil { + return "", 0, err } volSizeBytes := c.options.Capacity.Value() // Cinder works with gigabytes, convert to GiB with rounding up volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) - name, err := cloud.(*openstack.OpenStack).CreateVolume(volSizeGB) + name, err := cloud.CreateVolume(volSizeGB) if err != nil { glog.V(2).Infof("Error creating cinder volume: %v", err) return "", 0, err