Merge pull request #52767 from NickrenREN/rbd-resize

Automatic merge from submit-queue (batch tested with PRs 52767, 55065, 55148, 56228, 56221). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add resize support for ceph RBD

Add resize support for ceph RBD
**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: part of [#657](https://github.com/kubernetes/community/pull/657)

**Special notes for your reviewer**:

**Release note**:
```release-note
Add resize support for ceph RBD
```

WIP, need to add fs resize,
assign to myself first

/assign @NickrenREN

pkg/volume/rbd/BUILD

@@ -45,6 +45,7 @@ go_test(
         "//pkg/volume:go_default_library",
         "//pkg/volume/testing:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",

pkg/volume/rbd/disk_manager.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 )
@@ -46,6 +47,8 @@ type diskManager interface {
 	CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, volumeSizeGB int, err error)
 	// Deletes a rbd image.
 	DeleteImage(deleter *rbdVolumeDeleter) error
+	// Expands a rbd image
+	ExpandImage(expander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error)
 }
 
 // utility to mount a disk based filesystem

pkg/volume/rbd/rbd.go

@@ -54,6 +54,7 @@ var _ volume.PersistentVolumePlugin = &rbdPlugin{}
 var _ volume.DeletableVolumePlugin = &rbdPlugin{}
 var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
 var _ volume.AttachableVolumePlugin = &rbdPlugin{}
+var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
 
 const (
 	rbdPluginName                  = "kubernetes.io/rbd"
@@ -122,6 +123,85 @@ func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
 	}
 }
 
+type rbdVolumeExpander struct {
+	*rbdMounter
+}
+
+func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
+	class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
+	if err != nil {
+		return "", "", err
+	}
+	adminSecretName := ""
+	adminSecretNamespace := rbdDefaultAdminSecretNamespace
+	admin := ""
+
+	for k, v := range class.Parameters {
+		switch dstrings.ToLower(k) {
+		case "adminid":
+			admin = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		}
+	}
+
+	if admin == "" {
+		admin = rbdDefaultAdminId
+	}
+	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
+	if err != nil {
+		return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
+	}
+
+	return admin, secret, nil
+}
+
+func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
+		return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
+	}
+
+	// get admin and secret
+	admin, secret, err := plugin.getAdminAndSecret(spec)
+	if err != nil {
+		return oldSize, err
+	}
+
+	expander := &rbdVolumeExpander{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				volName: spec.Name(),
+				Image:   spec.PersistentVolume.Spec.RBD.RBDImage,
+				Pool:    spec.PersistentVolume.Spec.RBD.RBDPool,
+				plugin:  plugin,
+				manager: &RBDUtil{},
+				mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
+				exec:    plugin.host.GetExec(plugin.GetPluginName()),
+			},
+			Mon:         spec.PersistentVolume.Spec.RBD.CephMonitors,
+			adminId:     admin,
+			adminSecret: secret,
+		},
+	}
+
+	expandedSize, err := expander.ResizeImage(oldSize, newSize)
+	if err != nil {
+		return oldSize, err
+	} else {
+		return expandedSize, nil
+	}
+}
+
+func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
+	return expander.manager.ExpandImage(expander, oldSize, newSize)
+}
+
+func (plugin *rbdPlugin) RequiresFSResize() bool {
+	return true
+}
+
 func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) {
 	var err error
 	mon, err := getVolumeSourceMonitors(spec)
@@ -281,32 +361,12 @@ func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
 	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
 		return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
 	}
-	class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
+
+	admin, secret, err := plugin.getAdminAndSecret(spec)
 	if err != nil {
 		return nil, err
 	}
-	adminSecretName := ""
-	adminSecretNamespace := rbdDefaultAdminSecretNamespace
-	admin := ""
 
-	for k, v := range class.Parameters {
-		switch dstrings.ToLower(k) {
-		case "adminid":
-			admin = v
-		case "adminsecretname":
-			adminSecretName = v
-		case "adminsecretnamespace":
-			adminSecretNamespace = v
-		}
-	}
-
-	if admin == "" {
-		admin = rbdDefaultAdminId
-	}
-	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
-	if err != nil {
-		return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
-	}
 	return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
 }
 

pkg/volume/rbd/rbd_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -142,6 +143,10 @@ func (fake *fakeDiskManager) IsLocked(r rbdMounter, nodeName string) (bool, erro
 	return ok && isLocked, nil
 }
 
+func (fake *fakeDiskManager) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
+	return resource.Quantity{}, fmt.Errorf("not implemented")
+}
+
 // checkMounterLog checks fakeMounter must have expected logs, and the last action msut equal to expectedAction.
 func checkMounterLog(t *testing.T, fakeMounter *mount.FakeMounter, expected int, expectedAction mount.FakeAction) {
 	if len(fakeMounter.Log) != expected {

pkg/volume/rbd/rbd_util.go

@@ -30,11 +30,13 @@ import (
 	"os/exec"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	fileutil "k8s.io/kubernetes/pkg/util/file"
 	"k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/volume"
@@ -43,6 +45,8 @@ import (
 
 const (
 	imageWatcherStr = "watcher="
+	imageSizeStr    = "size "
+	sizeDivStr      = " MB in"
 	kubeLockMagic   = "kubelet_lock_magic_"
 )
 
@@ -441,6 +445,123 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
 	return fmt.Errorf("error %v, rbd output: %v", err, string(output))
 }
 
+// ExpandImage runs rbd resize command to resize the specified image
+func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
+	var output []byte
+	var err error
+	volSizeBytes := newSize.Value()
+	// convert to MB that rbd defaults on
+	sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
+	newVolSz := fmt.Sprintf("%d", sz)
+	newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
+
+	// check the current size of rbd image, if equals to or greater that the new request size, do nothing
+	curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
+	if infoErr != nil {
+		return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
+	}
+	if curSize >= sz {
+		return newSizeQuant, nil
+	}
+
+	// rbd resize
+	l := len(rbdExpander.rbdMounter.Mon)
+	// pick a mon randomly
+	start := rand.Int() % l
+	// iterate all monitors until resize succeeds.
+	for i := start; i < start+l; i++ {
+		mon := rbdExpander.rbdMounter.Mon[i%l]
+		glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
+		output, err = rbdExpander.exec.Run("rbd",
+			"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret)
+		if err == nil {
+			return newSizeQuant, nil
+		} else {
+			glog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
+		}
+	}
+	return oldSize, err
+}
+
+// rbdInfo runs `rbd info` command to get the current image size in MB
+func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
+	var err error
+	var output string
+	var cmd []byte
+
+	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
+	id := b.adminId
+	secret := b.adminSecret
+	if id == "" {
+		id = b.Id
+		secret = b.Secret
+	}
+
+	l := len(b.Mon)
+	start := rand.Int() % l
+	// iterate all hosts until rbd command succeeds.
+	for i := start; i < start+l; i++ {
+		mon := b.Mon[i%l]
+		// cmd "rbd info" get the image info with the following output:
+		//
+		// # image exists (exit=0)
+		// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08
+		//    size 1024 MB in 256 objects
+		//    order 22 (4096 kB objects)
+		// 	  block_name_prefix: rbd_data.1253ac238e1f29
+		//    format: 2
+		//    ...
+		//
+		//  rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 --format json
+		// {"name":"volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.1253ac238e1f29","format":2,"features":["layering","exclusive-lock","object-map","fast-diff","deep-flatten"],"flags":[]}
+		//
+		//
+		// # image does not exist (exit=2)
+		// rbd: error opening image 1234: (2) No such file or directory
+		//
+		glog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret)
+		cmd, err = b.exec.Run("rbd",
+			"info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret)
+		output = string(cmd)
+
+		// break if command succeeds
+		if err == nil {
+			break
+		}
+
+		if err, ok := err.(*exec.Error); ok {
+			if err.Err == exec.ErrNotFound {
+				glog.Errorf("rbd cmd not found")
+				// fail fast if command not found
+				return 0, err
+			}
+		}
+	}
+
+	// If command never succeed, returns its last error.
+	if err != nil {
+		return 0, err
+	}
+
+	if len(output) == 0 {
+		return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
+	}
+
+	// get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects`
+	sizeIndex := strings.Index(output, imageSizeStr)
+	divIndex := strings.Index(output, sizeDivStr)
+	if sizeIndex == -1 || divIndex == -1 || divIndex <= sizeIndex+5 {
+		return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
+	}
+	rbdSizeStr := output[sizeIndex+5 : divIndex]
+	rbdSize, err := strconv.Atoi(rbdSizeStr)
+	if err != nil {
+		return 0, fmt.Errorf("can not convert size str: %s to int", rbdSizeStr)
+	}
+
+	return rbdSize, nil
+}
+
 // rbdStatus runs `rbd status` command to check if there is watcher on the image.
 func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) {
 	var err error

plugin/pkg/admission/persistentvolume/resize/admission.go

@@ -149,10 +149,7 @@ func (pvcr *persistentVolumeClaimResize) allowResize(pvc, oldPvc *api.Persistent
 
 // checkVolumePlugin checks whether the volume plugin supports resize
 func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVolume) bool {
-	if pv.Spec.Glusterfs != nil {
+	if pv.Spec.Glusterfs != nil || pv.Spec.Cinder != nil || pv.Spec.RBD != nil {
-		return true
-	}
-	if pv.Spec.Cinder != nil {
 		return true
 	}