diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 12a2017fb33..b58fcda5dca 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -29,6 +29,7 @@ import ( // Volume plugins "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/aws_ebs" @@ -92,8 +93,8 @@ func NewVolumeProvisioner(cloud cloudprovider.Interface, flags VolumeConfigFlags return getProvisionablePluginFromVolumePlugins(aws_ebs.ProbeVolumePlugins()) // case cloud != nil && gce.ProviderName == cloud.ProviderName(): // return getProvisionablePluginFromVolumePlugins(gce_pd.ProbeVolumePlugins()) - // case cloud != nil && openstack.ProviderName == cloud.ProviderName(): - // return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins()) + case cloud != nil && openstack.ProviderName == cloud.ProviderName(): + return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins()) } return nil, nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 6dfbe9aae50..bfc34cd5410 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -1023,3 +1023,41 @@ func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) { } return volume, err } + +// Create a volume of given size (in GiB) +func (os *OpenStack) CreateVolume(size int) (volumeName string, err error) { + + sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{ + Region: os.region, + }) + + if err != nil || sClient == nil { + glog.Errorf("Unable to initialize cinder client for region: %s", os.region) + return "", err + } + + opts := volumes.CreateOpts{Size: size} + vol, err := volumes.Create(sClient, opts).Extract() + if err != nil { + glog.Errorf("Failed to create a %d GB volume: %v", size, err) + return "", err + } + glog.Infof("Created volume %v", vol.ID) + return vol.ID, err +} + +func (os *OpenStack) DeleteVolume(volumeName string) error { + sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{ + Region: os.region, + }) + + if err != nil || sClient == nil { + glog.Errorf("Unable to initialize cinder client for region: %s", os.region) + return err + } + err = volumes.Delete(sClient, volumeName).ExtractErr() + if err != nil { + glog.Errorf("Cannot delete volume %s: %v", volumeName, err) + } + return err +} diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go index 0c566767616..be70f496d02 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ b/pkg/cloudprovider/providers/openstack/openstack_test.go @@ -198,3 +198,26 @@ func TestZones(t *testing.T) { t.Fatalf("GetZone() returned wrong region (%s)", zone.Region) } } + +func TestVolumes(t *testing.T) { + cfg, ok := configFromEnv() + if !ok { + t.Skipf("No config found in environment") + } + + os, err := newOpenStack(cfg) + if err != nil { + t.Fatalf("Failed to construct/authenticate OpenStack: %s", err) + } + + vol, err := os.CreateVolume(1) + if err != nil { + t.Fatalf("Cannot create a new Cinder volume: %v", err) + } + + err = os.DeleteVolume(vol) + if err != nil { + t.Fatalf("Cannot delete Cinder volume %s: %v", vol, err) + } + +} diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index f1366a04831..1d62f696a86 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -17,11 +17,13 @@ limitations under the License. package cinder import ( + "fmt" "os" "path" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" @@ -39,6 +41,9 @@ type cinderPlugin struct { } var _ volume.VolumePlugin = &cinderPlugin{} +var _ volume.PersistentVolumePlugin = &cinderPlugin{} +var _ volume.DeletableVolumePlugin = &cinderPlugin{} +var _ volume.ProvisionableVolumePlugin = &cinderPlugin{} const ( cinderVolumePluginName = "kubernetes.io/cinder" @@ -107,12 +112,50 @@ func (plugin *cinderPlugin) newCleanerInternal(volName string, podUID types.UID, }}, nil } +func (plugin *cinderPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + return plugin.newDeleterInternal(spec, &CinderDiskUtil{}) +} + +func (plugin *cinderPlugin) newDeleterInternal(spec *volume.Spec, manager cdManager) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Cinder == nil { + return nil, fmt.Errorf("spec.PersistentVolumeSource.Cinder is nil") + } + return &cinderVolumeDeleter{ + &cinderVolume{ + volName: spec.Name(), + pdName: spec.PersistentVolume.Spec.Cinder.VolumeID, + manager: manager, + plugin: plugin, + }}, nil +} + +func (plugin *cinderPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + if len(options.AccessModes) == 0 { + options.AccessModes = plugin.GetAccessModes() + } + return plugin.newProvisionerInternal(options, &CinderDiskUtil{}) +} + +func (plugin *cinderPlugin) newProvisionerInternal(options volume.VolumeOptions, manager cdManager) (volume.Provisioner, error) { + return &cinderVolumeProvisioner{ + cinderVolume: &cinderVolume{ + manager: manager, + plugin: plugin, + }, + options: options, + }, nil +} + // Abstract interface to PD operations. type cdManager interface { // Attaches the disk to the kubelet's host machine. AttachDisk(builder *cinderVolumeBuilder, globalPDPath string) error // Detaches the disk from the kubelet's host machine. DetachDisk(cleaner *cinderVolumeCleaner) error + // Creates a volume + CreateVolume(provisioner *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) + // Deletes a volume + DeleteVolume(deleter *cinderVolumeDeleter) error } var _ volume.Builder = &cinderVolumeBuilder{} @@ -285,3 +328,66 @@ func (c *cinderVolumeCleaner) TearDownAt(dir string) error { } return nil } + +type cinderVolumeDeleter struct { + *cinderVolume +} + +var _ volume.Deleter = &cinderVolumeDeleter{} + +func (r *cinderVolumeDeleter) GetPath() string { + name := cinderVolumePluginName + return r.plugin.host.GetPodVolumeDir(r.podUID, util.EscapeQualifiedNameForDisk(name), r.volName) +} + +func (r *cinderVolumeDeleter) Delete() error { + return r.manager.DeleteVolume(r) +} + +type cinderVolumeProvisioner struct { + *cinderVolume + options volume.VolumeOptions +} + +var _ volume.Provisioner = &cinderVolumeProvisioner{} + +func (c *cinderVolumeProvisioner) Provision(pv *api.PersistentVolume) error { + volumeID, sizeGB, err := c.manager.CreateVolume(c) + if err != nil { + return err + } + pv.Spec.PersistentVolumeSource.Cinder.VolumeID = volumeID + pv.Spec.Capacity = api.ResourceList{ + api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)), + } + return nil +} + +func (c *cinderVolumeProvisioner) NewPersistentVolumeTemplate() (*api.PersistentVolume, error) { + // Provide dummy api.PersistentVolume.Spec, it will be filled in + // cinderVolumeProvisioner.Provision() + return &api.PersistentVolume{ + ObjectMeta: api.ObjectMeta{ + GenerateName: "pv-cinder-", + Labels: map[string]string{}, + Annotations: map[string]string{ + "kubernetes.io/createdby": "cinder-dynamic-provisioner", + }, + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy, + AccessModes: c.options.AccessModes, + Capacity: api.ResourceList{ + api.ResourceName(api.ResourceStorage): c.options.Capacity, + }, + PersistentVolumeSource: api.PersistentVolumeSource{ + Cinder: &api.CinderVolumeSource{ + VolumeID: "dummy", + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + }, nil + +} diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index c89ad0e55a1..e22519f10d4 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -17,12 +17,14 @@ limitations under the License. package cinder import ( + "fmt" "io/ioutil" "os" "path" "testing" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -73,6 +75,17 @@ func (fake *fakePDManager) DetachDisk(c *cinderVolumeCleaner) error { return nil } +func (fake *fakePDManager) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) { + return "test-volume-name", 1, nil +} + +func (fake *fakePDManager) DeleteVolume(cd *cinderVolumeDeleter) error { + if cd.pdName != "test-volume-name" { + return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdName) + } + return nil +} + func TestPlugin(t *testing.T) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cinderTest") if err != nil { @@ -142,4 +155,45 @@ func TestPlugin(t *testing.T) { } else if !os.IsNotExist(err) { t.Errorf("SetUp() failed: %v", err) } + + // Test Provisioner + cap := resource.MustParse("100Mi") + options := volume.VolumeOptions{ + Capacity: cap, + AccessModes: []api.PersistentVolumeAccessMode{ + api.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete, + } + provisioner, err := plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{}) + persistentSpec, err := provisioner.NewPersistentVolumeTemplate() + if err != nil { + t.Errorf("NewPersistentVolumeTemplate() failed: %v", err) + } + + // get 2nd Provisioner - persistent volume controller will do the same + provisioner, err = plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{}) + err = provisioner.Provision(persistentSpec) + if err != nil { + t.Errorf("Provision() failed: %v", err) + } + + if persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID != "test-volume-name" { + t.Errorf("Provision() returned unexpected volume ID: %s", persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID) + } + cap = persistentSpec.Spec.Capacity[api.ResourceStorage] + size := cap.Value() + if size != 1024*1024*1024 { + t.Errorf("Provision() returned unexpected volume size: %v", size) + } + + // Test Deleter + volSpec := &volume.Spec{ + PersistentVolume: persistentSpec, + } + deleter, err := plug.(*cinderPlugin).newDeleterInternal(volSpec, &fakePDManager{}) + err = deleter.Delete() + if err != nil { + t.Errorf("Deleter() failed: %v", err) + } } diff --git a/pkg/volume/cinder/cinder_util.go b/pkg/volume/cinder/cinder_util.go index 685333a436e..22aef4cbe19 100644 --- a/pkg/volume/cinder/cinder_util.go +++ b/pkg/volume/cinder/cinder_util.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" ) type CinderDiskUtil struct{} @@ -132,6 +133,40 @@ func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeCleaner) error { return nil } +func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error { + cloud := cd.plugin.host.GetCloudProvider() + if cloud == nil { + glog.Errorf("Cloud provider not initialized properly") + return errors.New("Cloud provider not initialized properly") + } + + if err := cloud.(*openstack.OpenStack).DeleteVolume(cd.pdName); err != nil { + glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err) + return err + } + glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName) + return nil +} + +func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) { + cloud := c.plugin.host.GetCloudProvider() + if cloud == nil { + glog.Errorf("Cloud provider not initialized properly") + return "", 0, errors.New("Cloud provider not initialized properly") + } + + volSizeBytes := c.options.Capacity.Value() + // Cinder works with gigabytes, convert to GiB with rounding up + volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) + name, err := cloud.(*openstack.OpenStack).CreateVolume(volSizeGB) + if err != nil { + glog.V(2).Infof("Error creating cinder volume: %v", err) + return "", 0, err + } + glog.V(2).Infof("Successfully created cinder volume %s", name) + return name, volSizeGB, nil +} + type cinderSafeFormatAndMount struct { mount.Interface runner exec.Interface