From 71406aa4a63bb7ae165fdca73313f6c1bac79578 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 16 Feb 2017 15:55:55 -0500 Subject: [PATCH] force rbd image unlock if the image is not used Signed-off-by: Huamin Chen --- pkg/volume/rbd/rbd_util.go | 83 +++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index ed5c0cd084c..5921432785d 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -43,6 +43,7 @@ import ( const ( imageWatcherStr = "watcher=" + kubeLockMagic = "kubelet_lock_magic_" ) // search /sys/bus for rbd device that matches given pool and image @@ -114,8 +115,15 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { } else { secret_opt = []string{"-k", b.Keyring} } + if len(b.adminId) == 0 { + b.adminId = b.Id + } + if len(b.adminSecret) == 0 { + b.adminSecret = b.Secret + } + // construct lock id using host name and a magic prefix - lock_id := "kubelet_lock_magic_" + node.GetHostname("") + lock_id := kubeLockMagic + node.GetHostname("") l := len(b.Mon) // avoid mount storm, pick a host randomly @@ -130,7 +138,7 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { cmd, err = b.plugin.execCommand("rbd", append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) output = string(cmd) - + glog.Infof("lock list output %q", output) if err != nil { continue } @@ -142,6 +150,23 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { glog.V(1).Infof("rbd: lock already held for %s", lock_id) return nil } + // clean up orphaned lock if no watcher on the image + used, statusErr := util.rbdStatus(&b) + if statusErr == nil && !used { + re := regexp.MustCompile("client.* " + kubeLockMagic + ".*") + locks := re.FindAllStringSubmatch(output, -1) + for _, v := range locks { + if len(v) > 0 { + lockInfo := strings.Split(v[0], " ") + if len(lockInfo) > 2 { + cmd, err = b.plugin.execCommand("rbd", + append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd)) + } + } + } + } + // hold a lock: rbd lock add cmd, err = b.plugin.execCommand("rbd", append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) @@ -220,6 +245,20 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { var err error var output []byte + // create mount point + globalPDPath := b.manager.MakeGlobalPDName(*b.rbd) + notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath) + // in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath) + } + if !notMnt { + return nil + } + if err = os.MkdirAll(globalPDPath, 0750); err != nil { + return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath) + } + devicePath, found := waitForPath(b.Pool, b.Image, 1) if !found { // modprobe @@ -227,6 +266,18 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { if err != nil { return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err) } + + // fence off other mappers + if err = util.fencing(b); err != nil { + return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image) + } + // rbd lock remove needs ceph and image config + // but kubelet doesn't get them from apiserver during teardown + // so persit rbd config so upon disk detach, rbd lock can be removed + // since rbd json is persisted in the same local directory that is used as rbd mountpoint later, + // the json file remains invisible during rbd mount and thus won't be removed accidentally. + util.persistRBD(b, globalPDPath) + // rbd map l := len(b.Mon) // avoid mount storm, pick a host randomly @@ -255,34 +306,8 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { return errors.New("Could not map image: Timeout after 10s") } } + // mount it - globalPDPath := b.manager.MakeGlobalPDName(*b.rbd) - notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath) - // in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath) - } - if !notMnt { - return nil - } - - if err := os.MkdirAll(globalPDPath, 0750); err != nil { - return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath) - } - - // fence off other mappers - if err := util.fencing(b); err != nil { - // rbd unmap before exit - b.plugin.execCommand("rbd", []string{"unmap", devicePath}) - return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image) - } - // rbd lock remove needs ceph and image config - // but kubelet doesn't get them from apiserver during teardown - // so persit rbd config so upon disk detach, rbd lock can be removed - // since rbd json is persisted in the same local directory that is used as rbd mountpoint later, - // the json file remains invisible during rbd mount and thus won't be removed accidentally. - util.persistRBD(b, globalPDPath) - if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil { err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) }