mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #52767 from NickrenREN/rbd-resize
Automatic merge from submit-queue (batch tested with PRs 52767, 55065, 55148, 56228, 56221). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add resize support for ceph RBD Add resize support for ceph RBD **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: part of [#657](https://github.com/kubernetes/community/pull/657) **Special notes for your reviewer**: **Release note**: ```release-note Add resize support for ceph RBD ``` WIP, need to add fs resize, assign to myself first /assign @NickrenREN
This commit is contained in:
commit
db2a08bc32
@ -45,6 +45,7 @@ go_test(
|
||||
"//pkg/volume:go_default_library",
|
||||
"//pkg/volume/testing:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
|
||||
|
@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
)
|
||||
@ -46,6 +47,8 @@ type diskManager interface {
|
||||
CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, volumeSizeGB int, err error)
|
||||
// Deletes a rbd image.
|
||||
DeleteImage(deleter *rbdVolumeDeleter) error
|
||||
// Expands a rbd image
|
||||
ExpandImage(expander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error)
|
||||
}
|
||||
|
||||
// utility to mount a disk based filesystem
|
||||
|
@ -54,6 +54,7 @@ var _ volume.PersistentVolumePlugin = &rbdPlugin{}
|
||||
var _ volume.DeletableVolumePlugin = &rbdPlugin{}
|
||||
var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
|
||||
var _ volume.AttachableVolumePlugin = &rbdPlugin{}
|
||||
var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
|
||||
|
||||
const (
|
||||
rbdPluginName = "kubernetes.io/rbd"
|
||||
@ -122,6 +123,85 @@ func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
|
||||
}
|
||||
}
|
||||
|
||||
type rbdVolumeExpander struct {
|
||||
*rbdMounter
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
|
||||
class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
adminSecretName := ""
|
||||
adminSecretNamespace := rbdDefaultAdminSecretNamespace
|
||||
admin := ""
|
||||
|
||||
for k, v := range class.Parameters {
|
||||
switch dstrings.ToLower(k) {
|
||||
case "adminid":
|
||||
admin = v
|
||||
case "adminsecretname":
|
||||
adminSecretName = v
|
||||
case "adminsecretnamespace":
|
||||
adminSecretNamespace = v
|
||||
}
|
||||
}
|
||||
|
||||
if admin == "" {
|
||||
admin = rbdDefaultAdminId
|
||||
}
|
||||
secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
|
||||
if err != nil {
|
||||
return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
|
||||
}
|
||||
|
||||
return admin, secret, nil
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
|
||||
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
|
||||
return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
|
||||
}
|
||||
|
||||
// get admin and secret
|
||||
admin, secret, err := plugin.getAdminAndSecret(spec)
|
||||
if err != nil {
|
||||
return oldSize, err
|
||||
}
|
||||
|
||||
expander := &rbdVolumeExpander{
|
||||
rbdMounter: &rbdMounter{
|
||||
rbd: &rbd{
|
||||
volName: spec.Name(),
|
||||
Image: spec.PersistentVolume.Spec.RBD.RBDImage,
|
||||
Pool: spec.PersistentVolume.Spec.RBD.RBDPool,
|
||||
plugin: plugin,
|
||||
manager: &RBDUtil{},
|
||||
mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
|
||||
exec: plugin.host.GetExec(plugin.GetPluginName()),
|
||||
},
|
||||
Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
|
||||
adminId: admin,
|
||||
adminSecret: secret,
|
||||
},
|
||||
}
|
||||
|
||||
expandedSize, err := expander.ResizeImage(oldSize, newSize)
|
||||
if err != nil {
|
||||
return oldSize, err
|
||||
} else {
|
||||
return expandedSize, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
|
||||
return expander.manager.ExpandImage(expander, oldSize, newSize)
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) RequiresFSResize() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) {
|
||||
var err error
|
||||
mon, err := getVolumeSourceMonitors(spec)
|
||||
@ -281,32 +361,12 @@ func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
|
||||
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
|
||||
return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
|
||||
}
|
||||
class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
|
||||
|
||||
admin, secret, err := plugin.getAdminAndSecret(spec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
adminSecretName := ""
|
||||
adminSecretNamespace := rbdDefaultAdminSecretNamespace
|
||||
admin := ""
|
||||
|
||||
for k, v := range class.Parameters {
|
||||
switch dstrings.ToLower(k) {
|
||||
case "adminid":
|
||||
admin = v
|
||||
case "adminsecretname":
|
||||
adminSecretName = v
|
||||
case "adminsecretnamespace":
|
||||
adminSecretNamespace = v
|
||||
}
|
||||
}
|
||||
|
||||
if admin == "" {
|
||||
admin = rbdDefaultAdminId
|
||||
}
|
||||
secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
|
||||
}
|
||||
return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
|
||||
}
|
||||
|
||||
|
@ -27,6 +27,7 @@ import (
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
@ -142,6 +143,10 @@ func (fake *fakeDiskManager) IsLocked(r rbdMounter, nodeName string) (bool, erro
|
||||
return ok && isLocked, nil
|
||||
}
|
||||
|
||||
func (fake *fakeDiskManager) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
|
||||
return resource.Quantity{}, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
// checkMounterLog checks fakeMounter must have expected logs, and the last action msut equal to expectedAction.
|
||||
func checkMounterLog(t *testing.T, fakeMounter *mount.FakeMounter, expected int, expectedAction mount.FakeAction) {
|
||||
if len(fakeMounter.Log) != expected {
|
||||
|
@ -30,11 +30,13 @@ import (
|
||||
"os/exec"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
fileutil "k8s.io/kubernetes/pkg/util/file"
|
||||
"k8s.io/kubernetes/pkg/util/node"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
@ -43,6 +45,8 @@ import (
|
||||
|
||||
const (
|
||||
imageWatcherStr = "watcher="
|
||||
imageSizeStr = "size "
|
||||
sizeDivStr = " MB in"
|
||||
kubeLockMagic = "kubelet_lock_magic_"
|
||||
)
|
||||
|
||||
@ -441,6 +445,123 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
|
||||
return fmt.Errorf("error %v, rbd output: %v", err, string(output))
|
||||
}
|
||||
|
||||
// ExpandImage runs rbd resize command to resize the specified image
|
||||
func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
|
||||
var output []byte
|
||||
var err error
|
||||
volSizeBytes := newSize.Value()
|
||||
// convert to MB that rbd defaults on
|
||||
sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
|
||||
newVolSz := fmt.Sprintf("%d", sz)
|
||||
newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
|
||||
|
||||
// check the current size of rbd image, if equals to or greater that the new request size, do nothing
|
||||
curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
|
||||
if infoErr != nil {
|
||||
return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
|
||||
}
|
||||
if curSize >= sz {
|
||||
return newSizeQuant, nil
|
||||
}
|
||||
|
||||
// rbd resize
|
||||
l := len(rbdExpander.rbdMounter.Mon)
|
||||
// pick a mon randomly
|
||||
start := rand.Int() % l
|
||||
// iterate all monitors until resize succeeds.
|
||||
for i := start; i < start+l; i++ {
|
||||
mon := rbdExpander.rbdMounter.Mon[i%l]
|
||||
glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
|
||||
output, err = rbdExpander.exec.Run("rbd",
|
||||
"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret)
|
||||
if err == nil {
|
||||
return newSizeQuant, nil
|
||||
} else {
|
||||
glog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
|
||||
}
|
||||
}
|
||||
return oldSize, err
|
||||
}
|
||||
|
||||
// rbdInfo runs `rbd info` command to get the current image size in MB
|
||||
func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
|
||||
var err error
|
||||
var output string
|
||||
var cmd []byte
|
||||
|
||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||
id := b.adminId
|
||||
secret := b.adminSecret
|
||||
if id == "" {
|
||||
id = b.Id
|
||||
secret = b.Secret
|
||||
}
|
||||
|
||||
l := len(b.Mon)
|
||||
start := rand.Int() % l
|
||||
// iterate all hosts until rbd command succeeds.
|
||||
for i := start; i < start+l; i++ {
|
||||
mon := b.Mon[i%l]
|
||||
// cmd "rbd info" get the image info with the following output:
|
||||
//
|
||||
// # image exists (exit=0)
|
||||
// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08
|
||||
// size 1024 MB in 256 objects
|
||||
// order 22 (4096 kB objects)
|
||||
// block_name_prefix: rbd_data.1253ac238e1f29
|
||||
// format: 2
|
||||
// ...
|
||||
//
|
||||
// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 --format json
|
||||
// {"name":"volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.1253ac238e1f29","format":2,"features":["layering","exclusive-lock","object-map","fast-diff","deep-flatten"],"flags":[]}
|
||||
//
|
||||
//
|
||||
// # image does not exist (exit=2)
|
||||
// rbd: error opening image 1234: (2) No such file or directory
|
||||
//
|
||||
glog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret)
|
||||
cmd, err = b.exec.Run("rbd",
|
||||
"info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret)
|
||||
output = string(cmd)
|
||||
|
||||
// break if command succeeds
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
if err, ok := err.(*exec.Error); ok {
|
||||
if err.Err == exec.ErrNotFound {
|
||||
glog.Errorf("rbd cmd not found")
|
||||
// fail fast if command not found
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If command never succeed, returns its last error.
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if len(output) == 0 {
|
||||
return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
|
||||
}
|
||||
|
||||
// get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects`
|
||||
sizeIndex := strings.Index(output, imageSizeStr)
|
||||
divIndex := strings.Index(output, sizeDivStr)
|
||||
if sizeIndex == -1 || divIndex == -1 || divIndex <= sizeIndex+5 {
|
||||
return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
|
||||
}
|
||||
rbdSizeStr := output[sizeIndex+5 : divIndex]
|
||||
rbdSize, err := strconv.Atoi(rbdSizeStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("can not convert size str: %s to int", rbdSizeStr)
|
||||
}
|
||||
|
||||
return rbdSize, nil
|
||||
}
|
||||
|
||||
// rbdStatus runs `rbd status` command to check if there is watcher on the image.
|
||||
func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) {
|
||||
var err error
|
||||
|
@ -149,10 +149,7 @@ func (pvcr *persistentVolumeClaimResize) allowResize(pvc, oldPvc *api.Persistent
|
||||
|
||||
// checkVolumePlugin checks whether the volume plugin supports resize
|
||||
func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVolume) bool {
|
||||
if pv.Spec.Glusterfs != nil {
|
||||
return true
|
||||
}
|
||||
if pv.Spec.Cinder != nil {
|
||||
if pv.Spec.Glusterfs != nil || pv.Spec.Cinder != nil || pv.Spec.RBD != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user