github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 20:24:09 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #52767 from NickrenREN/rbd-resize

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 52767, 55065, 55148, 56228, 56221). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add resize support for ceph RBD

Add resize support for ceph RBD
**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: part of [#657](https://github.com/kubernetes/community/pull/657)

**Special notes for your reviewer**:

**Release note**:
```release-note
Add resize support for ceph RBD
```

WIP, need to add fs resize,
assign to myself first

/assign @NickrenREN
This commit is contained in:
Kubernetes Submit Queue 2017-11-22 19:49:32 -08:00 committed by GitHub
commit db2a08bc32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 213 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/volume/rbd/BUILD
View File

@ -45,6 +45,7 @@ go_test(
"//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",

3
pkg/volume/rbd/disk_manager.go
View File

@ -28,6 +28,7 @@ import (
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
)
@ -46,6 +47,8 @@ type diskManager interface {
CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, volumeSizeGB int, err error)
// Deletes a rbd image.
DeleteImage(deleter *rbdVolumeDeleter) error
// Expands a rbd image
ExpandImage(expander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error)
}
// utility to mount a disk based filesystem

104
pkg/volume/rbd/rbd.go
View File

@ -54,6 +54,7 @@ var _ volume.PersistentVolumePlugin = &rbdPlugin{}
var _ volume.DeletableVolumePlugin = &rbdPlugin{}
var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
var _ volume.AttachableVolumePlugin = &rbdPlugin{}
var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
const (
rbdPluginName = "kubernetes.io/rbd"
@ -122,6 +123,85 @@ func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
}
}
type rbdVolumeExpander struct {
*rbdMounter
}
func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
if err != nil {
return "", "", err
}
adminSecretName := ""
adminSecretNamespace := rbdDefaultAdminSecretNamespace
admin := ""
for k, v := range class.Parameters {
switch dstrings.ToLower(k) {
case "adminid":
admin = v
case "adminsecretname":
adminSecretName = v
case "adminsecretnamespace":
adminSecretNamespace = v
}
}
if admin == "" {
admin = rbdDefaultAdminId
}
secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
if err != nil {
return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
}
return admin, secret, nil
}
func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
}
// get admin and secret
admin, secret, err := plugin.getAdminAndSecret(spec)
if err != nil {
return oldSize, err
}
expander := &rbdVolumeExpander{
rbdMounter: &rbdMounter{
rbd: &rbd{
volName: spec.Name(),
Image: spec.PersistentVolume.Spec.RBD.RBDImage,
Pool: spec.PersistentVolume.Spec.RBD.RBDPool,
plugin: plugin,
manager: &RBDUtil{},
mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
exec: plugin.host.GetExec(plugin.GetPluginName()),
},
Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
adminId: admin,
adminSecret: secret,
},
}
expandedSize, err := expander.ResizeImage(oldSize, newSize)
if err != nil {
return oldSize, err
} else {
return expandedSize, nil
}
}
func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
return expander.manager.ExpandImage(expander, oldSize, newSize)
}
func (plugin *rbdPlugin) RequiresFSResize() bool {
return true
}
func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) {
var err error
mon, err := getVolumeSourceMonitors(spec)
@ -281,32 +361,12 @@ func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
}
class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
admin, secret, err := plugin.getAdminAndSecret(spec)
if err != nil {
return nil, err
}
adminSecretName := ""
adminSecretNamespace := rbdDefaultAdminSecretNamespace
admin := ""
for k, v := range class.Parameters {
switch dstrings.ToLower(k) {
case "adminid":
admin = v
case "adminsecretname":
adminSecretName = v
case "adminsecretnamespace":
adminSecretNamespace = v
}
}
if admin == "" {
admin = rbdDefaultAdminId
}
secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
if err != nil {
return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
}
return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
}

5
pkg/volume/rbd/rbd_test.go
View File

@ -27,6 +27,7 @@ import (
"time"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
@ -142,6 +143,10 @@ func (fake *fakeDiskManager) IsLocked(r rbdMounter, nodeName string) (bool, erro
return ok && isLocked, nil
}
func (fake *fakeDiskManager) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
return resource.Quantity{}, fmt.Errorf("not implemented")
}
// checkMounterLog checks fakeMounter must have expected logs, and the last action msut equal to expectedAction.
func checkMounterLog(t *testing.T, fakeMounter *mount.FakeMounter, expected int, expectedAction mount.FakeAction) {
if len(fakeMounter.Log) != expected {

121
pkg/volume/rbd/rbd_util.go
View File

@ -30,11 +30,13 @@ import (
"os/exec"
"path"
"regexp"
"strconv"
"strings"
"time"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
fileutil "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/volume"
@ -43,6 +45,8 @@ import (
const (
imageWatcherStr = "watcher="
imageSizeStr = "size "
sizeDivStr = " MB in"
kubeLockMagic = "kubelet_lock_magic_"
)
@ -441,6 +445,123 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
return fmt.Errorf("error %v, rbd output: %v", err, string(output))
}
// ExpandImage runs rbd resize command to resize the specified image
func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
var output []byte
var err error
volSizeBytes := newSize.Value()
// convert to MB that rbd defaults on
sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
newVolSz := fmt.Sprintf("%d", sz)
newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
// check the current size of rbd image, if equals to or greater that the new request size, do nothing
curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
if infoErr != nil {
return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
}
if curSize >= sz {
return newSizeQuant, nil
}
// rbd resize
l := len(rbdExpander.rbdMounter.Mon)
// pick a mon randomly
start := rand.Int() % l
// iterate all monitors until resize succeeds.
for i := start; i < start+l; i++ {
mon := rbdExpander.rbdMounter.Mon[i%l]
glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
output, err = rbdExpander.exec.Run("rbd",
"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret)
if err == nil {
return newSizeQuant, nil
} else {
glog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
}
}
return oldSize, err
}
// rbdInfo runs `rbd info` command to get the current image size in MB
func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
var err error
var output string
var cmd []byte
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
id := b.adminId
secret := b.adminSecret
if id == "" {
id = b.Id
secret = b.Secret
}
l := len(b.Mon)
start := rand.Int() % l
// iterate all hosts until rbd command succeeds.
for i := start; i < start+l; i++ {
mon := b.Mon[i%l]
// cmd "rbd info" get the image info with the following output:
//
// # image exists (exit=0)
// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08
// size 1024 MB in 256 objects
// order 22 (4096 kB objects)
// block_name_prefix: rbd_data.1253ac238e1f29
// format: 2
// ...
//
// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 --format json
// {"name":"volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.1253ac238e1f29","format":2,"features":["layering","exclusive-lock","object-map","fast-diff","deep-flatten"],"flags":[]}
//
//
// # image does not exist (exit=2)
// rbd: error opening image 1234: (2) No such file or directory
//
glog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret)
cmd, err = b.exec.Run("rbd",
"info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret)
output = string(cmd)
// break if command succeeds
if err == nil {
break
}
if err, ok := err.(*exec.Error); ok {
if err.Err == exec.ErrNotFound {
glog.Errorf("rbd cmd not found")
// fail fast if command not found
return 0, err
}
}
}
// If command never succeed, returns its last error.
if err != nil {
return 0, err
}
if len(output) == 0 {
return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
}
// get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects`
sizeIndex := strings.Index(output, imageSizeStr)
divIndex := strings.Index(output, sizeDivStr)
if sizeIndex == -1 || divIndex == -1 || divIndex <= sizeIndex+5 {
return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
}
rbdSizeStr := output[sizeIndex+5 : divIndex]
rbdSize, err := strconv.Atoi(rbdSizeStr)
if err != nil {
return 0, fmt.Errorf("can not convert size str: %s to int", rbdSizeStr)
}
return rbdSize, nil
}
// rbdStatus runs `rbd status` command to check if there is watcher on the image.
func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) {
var err error

5
plugin/pkg/admission/persistentvolume/resize/admission.go
View File

@ -149,10 +149,7 @@ func (pvcr *persistentVolumeClaimResize) allowResize(pvc, oldPvc *api.Persistent
// checkVolumePlugin checks whether the volume plugin supports resize
func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVolume) bool {
if pv.Spec.Glusterfs != nil {
return true
}
if pv.Spec.Cinder != nil {
if pv.Spec.Glusterfs != nil || pv.Spec.Cinder != nil || pv.Spec.RBD != nil {
return true
}