From 6f7a1983eff67e950ffb457d46d4c84887120876 Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Wed, 20 Sep 2017 14:33:27 +0800 Subject: [PATCH] Support ceph rbd resize --- pkg/volume/rbd/BUILD | 1 + pkg/volume/rbd/disk_manager.go | 3 + pkg/volume/rbd/rbd.go | 104 +++++++++++---- pkg/volume/rbd/rbd_test.go | 5 + pkg/volume/rbd/rbd_util.go | 121 ++++++++++++++++++ .../persistentvolume/resize/admission.go | 5 +- 6 files changed, 213 insertions(+), 26 deletions(-) diff --git a/pkg/volume/rbd/BUILD b/pkg/volume/rbd/BUILD index aef8b366b4e..e97b0fc7567 100644 --- a/pkg/volume/rbd/BUILD +++ b/pkg/volume/rbd/BUILD @@ -45,6 +45,7 @@ go_test( "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", diff --git a/pkg/volume/rbd/disk_manager.go b/pkg/volume/rbd/disk_manager.go index 3ba83fe36cb..87607370448 100644 --- a/pkg/volume/rbd/disk_manager.go +++ b/pkg/volume/rbd/disk_manager.go @@ -28,6 +28,7 @@ import ( "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" ) @@ -46,6 +47,8 @@ type diskManager interface { CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, volumeSizeGB int, err error) // Deletes a rbd image. DeleteImage(deleter *rbdVolumeDeleter) error + // Expands a rbd image + ExpandImage(expander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) } // utility to mount a disk based filesystem diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 596a19dca8b..fcdb4fa5ee2 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -54,6 +54,7 @@ var _ volume.PersistentVolumePlugin = &rbdPlugin{} var _ volume.DeletableVolumePlugin = &rbdPlugin{} var _ volume.ProvisionableVolumePlugin = &rbdPlugin{} var _ volume.AttachableVolumePlugin = &rbdPlugin{} +var _ volume.ExpandableVolumePlugin = &rbdPlugin{} const ( rbdPluginName = "kubernetes.io/rbd" @@ -122,6 +123,85 @@ func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { } } +type rbdVolumeExpander struct { + *rbdMounter +} + +func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) { + class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume) + if err != nil { + return "", "", err + } + adminSecretName := "" + adminSecretNamespace := rbdDefaultAdminSecretNamespace + admin := "" + + for k, v := range class.Parameters { + switch dstrings.ToLower(k) { + case "adminid": + admin = v + case "adminsecretname": + adminSecretName = v + case "adminsecretnamespace": + adminSecretNamespace = v + } + } + + if admin == "" { + admin = rbdDefaultAdminId + } + secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient()) + if err != nil { + return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err) + } + + return admin, secret, nil +} + +func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil { + return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil") + } + + // get admin and secret + admin, secret, err := plugin.getAdminAndSecret(spec) + if err != nil { + return oldSize, err + } + + expander := &rbdVolumeExpander{ + rbdMounter: &rbdMounter{ + rbd: &rbd{ + volName: spec.Name(), + Image: spec.PersistentVolume.Spec.RBD.RBDImage, + Pool: spec.PersistentVolume.Spec.RBD.RBDPool, + plugin: plugin, + manager: &RBDUtil{}, + mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())}, + exec: plugin.host.GetExec(plugin.GetPluginName()), + }, + Mon: spec.PersistentVolume.Spec.RBD.CephMonitors, + adminId: admin, + adminSecret: secret, + }, + } + + expandedSize, err := expander.ResizeImage(oldSize, newSize) + if err != nil { + return oldSize, err + } else { + return expandedSize, nil + } +} + +func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { + return expander.manager.ExpandImage(expander, oldSize, newSize) +} + +func (plugin *rbdPlugin) RequiresFSResize() bool { + return true +} + func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) { var err error mon, err := getVolumeSourceMonitors(spec) @@ -281,32 +361,12 @@ func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil { return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil") } - class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume) + + admin, secret, err := plugin.getAdminAndSecret(spec) if err != nil { return nil, err } - adminSecretName := "" - adminSecretNamespace := rbdDefaultAdminSecretNamespace - admin := "" - for k, v := range class.Parameters { - switch dstrings.ToLower(k) { - case "adminid": - admin = v - case "adminsecretname": - adminSecretName = v - case "adminsecretnamespace": - adminSecretNamespace = v - } - } - - if admin == "" { - admin = rbdDefaultAdminId - } - secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient()) - if err != nil { - return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err) - } return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{}) } diff --git a/pkg/volume/rbd/rbd_test.go b/pkg/volume/rbd/rbd_test.go index a35a1a4e74d..d16ce140992 100644 --- a/pkg/volume/rbd/rbd_test.go +++ b/pkg/volume/rbd/rbd_test.go @@ -27,6 +27,7 @@ import ( "time" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" @@ -142,6 +143,10 @@ func (fake *fakeDiskManager) IsLocked(r rbdMounter, nodeName string) (bool, erro return ok && isLocked, nil } +func (fake *fakeDiskManager) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { + return resource.Quantity{}, fmt.Errorf("not implemented") +} + // checkMounterLog checks fakeMounter must have expected logs, and the last action msut equal to expectedAction. func checkMounterLog(t *testing.T, fakeMounter *mount.FakeMounter, expected int, expectedAction mount.FakeAction) { if len(fakeMounter.Log) != expected { diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index a7c27e08e86..15b4b52ee66 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -30,11 +30,13 @@ import ( "os/exec" "path" "regexp" + "strconv" "strings" "time" "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" fileutil "k8s.io/kubernetes/pkg/util/file" "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/volume" @@ -43,6 +45,8 @@ import ( const ( imageWatcherStr = "watcher=" + imageSizeStr = "size " + sizeDivStr = " MB in" kubeLockMagic = "kubelet_lock_magic_" ) @@ -441,6 +445,123 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error { return fmt.Errorf("error %v, rbd output: %v", err, string(output)) } +// ExpandImage runs rbd resize command to resize the specified image +func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { + var output []byte + var err error + volSizeBytes := newSize.Value() + // convert to MB that rbd defaults on + sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024)) + newVolSz := fmt.Sprintf("%d", sz) + newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz)) + + // check the current size of rbd image, if equals to or greater that the new request size, do nothing + curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter) + if infoErr != nil { + return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr) + } + if curSize >= sz { + return newSizeQuant, nil + } + + // rbd resize + l := len(rbdExpander.rbdMounter.Mon) + // pick a mon randomly + start := rand.Int() % l + // iterate all monitors until resize succeeds. + for i := start; i < start+l; i++ { + mon := rbdExpander.rbdMounter.Mon[i%l] + glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret) + output, err = rbdExpander.exec.Run("rbd", + "resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret) + if err == nil { + return newSizeQuant, nil + } else { + glog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output)) + } + } + return oldSize, err +} + +// rbdInfo runs `rbd info` command to get the current image size in MB +func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) { + var err error + var output string + var cmd []byte + + // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. + id := b.adminId + secret := b.adminSecret + if id == "" { + id = b.Id + secret = b.Secret + } + + l := len(b.Mon) + start := rand.Int() % l + // iterate all hosts until rbd command succeeds. + for i := start; i < start+l; i++ { + mon := b.Mon[i%l] + // cmd "rbd info" get the image info with the following output: + // + // # image exists (exit=0) + // rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 + // size 1024 MB in 256 objects + // order 22 (4096 kB objects) + // block_name_prefix: rbd_data.1253ac238e1f29 + // format: 2 + // ... + // + // rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 --format json + // {"name":"volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.1253ac238e1f29","format":2,"features":["layering","exclusive-lock","object-map","fast-diff","deep-flatten"],"flags":[]} + // + // + // # image does not exist (exit=2) + // rbd: error opening image 1234: (2) No such file or directory + // + glog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret) + cmd, err = b.exec.Run("rbd", + "info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret) + output = string(cmd) + + // break if command succeeds + if err == nil { + break + } + + if err, ok := err.(*exec.Error); ok { + if err.Err == exec.ErrNotFound { + glog.Errorf("rbd cmd not found") + // fail fast if command not found + return 0, err + } + } + } + + // If command never succeed, returns its last error. + if err != nil { + return 0, err + } + + if len(output) == 0 { + return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output) + } + + // get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects` + sizeIndex := strings.Index(output, imageSizeStr) + divIndex := strings.Index(output, sizeDivStr) + if sizeIndex == -1 || divIndex == -1 || divIndex <= sizeIndex+5 { + return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output) + } + rbdSizeStr := output[sizeIndex+5 : divIndex] + rbdSize, err := strconv.Atoi(rbdSizeStr) + if err != nil { + return 0, fmt.Errorf("can not convert size str: %s to int", rbdSizeStr) + } + + return rbdSize, nil +} + // rbdStatus runs `rbd status` command to check if there is watcher on the image. func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) { var err error diff --git a/plugin/pkg/admission/persistentvolume/resize/admission.go b/plugin/pkg/admission/persistentvolume/resize/admission.go index f5327ecf636..4bf7b9f998e 100644 --- a/plugin/pkg/admission/persistentvolume/resize/admission.go +++ b/plugin/pkg/admission/persistentvolume/resize/admission.go @@ -149,10 +149,7 @@ func (pvcr *persistentVolumeClaimResize) allowResize(pvc, oldPvc *api.Persistent // checkVolumePlugin checks whether the volume plugin supports resize func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVolume) bool { - if pv.Spec.Glusterfs != nil { - return true - } - if pv.Spec.Cinder != nil { + if pv.Spec.Glusterfs != nil || pv.Spec.Cinder != nil || pv.Spec.RBD != nil { return true } return false