Use rbd-nbd if present for rbd volume map and unmap operations, if rbd fails.

pkg/volume/rbd/BUILD

@@ -29,6 +29,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",

pkg/volume/rbd/rbd_util.go

@@ -35,6 +35,7 @@ import (
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	fileutil "k8s.io/kubernetes/pkg/util/file"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -55,38 +56,50 @@ const (
 	rbdImageWatcherSteps     = 10
 )
 
-// search /sys/bus for rbd device that matches given pool and image
 func getDevFromImageAndPool(pool, image string) (string, bool) {
+	device, found := getRbdDevFromImageAndPool(pool, image)
+	if found {
+		return device, true
+	}
+	device, found = getNbdDevFromImageAndPool(pool, image)
+	if found {
+		return device, true
+	}
+	return "", false
+}
+
+// Search /sys/bus for rbd device that matches given pool and image.
+func getRbdDevFromImageAndPool(pool string, image string) (string, bool) {
 	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
 	sys_path := "/sys/bus/rbd/devices"
 	if dirs, err := ioutil.ReadDir(sys_path); err == nil {
 		for _, f := range dirs {
-			// pool and name format:
+			// Pool and name format:
 			// see rbd_pool_show() and rbd_name_show() at
 			// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
 			name := f.Name()
-			// first match pool, then match name
+			// First match pool, then match name.
 			poolFile := path.Join(sys_path, name, "pool")
 			poolBytes, err := ioutil.ReadFile(poolFile)
 			if err != nil {
-				glog.V(4).Infof("Error reading %s: %v", poolFile, err)
+				glog.V(4).Infof("error reading %s: %v", poolFile, err)
 				continue
 			}
 			if strings.TrimSpace(string(poolBytes)) != pool {
-				glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes))
+				glog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes))
 				continue
 			}
 			imgFile := path.Join(sys_path, name, "name")
 			imgBytes, err := ioutil.ReadFile(imgFile)
 			if err != nil {
-				glog.V(4).Infof("Error reading %s: %v", imgFile, err)
+				glog.V(4).Infof("error reading %s: %v", imgFile, err)
 				continue
 			}
 			if strings.TrimSpace(string(imgBytes)) != image {
-				glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes))
+				glog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes))
 				continue
 			}
-			// found a match, check if device exists
+			// Found a match, check if device exists.
 			devicePath := "/dev/rbd" + name
 			if _, err := os.Lstat(devicePath); err == nil {
 				return devicePath, true
@@ -96,24 +109,144 @@ func getDevFromImageAndPool(pool, image string) (string, bool) {
 	return "", false
 }
 
-// stat a path, if not exists, retry maxRetries times
+func getMaxNbds() (int, error) {
-func waitForPath(pool, image string, maxRetries int) (string, bool) {
+
-	for i := 0; i < maxRetries; i++ {
+	// the max number of nbd devices may be found in maxNbdsPath
-		devicePath, found := getDevFromImageAndPool(pool, image)
+	// we will check sysfs for possible nbd devices even if this is not available
-		if found {
+	maxNbdsPath := "/sys/module/nbd/parameters/nbds_max"
+	_, err := os.Lstat(maxNbdsPath)
+	if err != nil {
+		return 0, fmt.Errorf("rbd-nbd: failed to retrieve max_nbds from %s err: %q", maxNbdsPath, err)
+	}
+
+	glog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath)
+
+	maxNbdBytes, err := ioutil.ReadFile(maxNbdsPath)
+	if err != nil {
+		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds from %s err: %q", maxNbdsPath, err)
+	}
+
+	maxNbds, err := strconv.Atoi(strings.TrimSpace(string(maxNbdBytes)))
+	if err != nil {
+		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds err: %q", err)
+	}
+
+	glog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds)
+	return maxNbds, nil
+}
+
+// Locate any existing rbd-nbd process mapping given a <pool, image>.
+// Recent versions of rbd-nbd tool can correctly provide this info using list-mapped
+// but older versions of list-mapped don't.
+// The implementation below peeks at the command line of nbd bound processes
+// to figure out any mapped images.
+func getNbdDevFromImageAndPool(pool string, image string) (string, bool) {
+	// nbd module exports the pid of serving process in sysfs
+	basePath := "/sys/block/nbd"
+	// Do not change imgPath format - some tools like rbd-nbd are strict about it.
+	imgPath := fmt.Sprintf("%s/%s", pool, image)
+
+	maxNbds, maxNbdsErr := getMaxNbds()
+	if maxNbdsErr != nil {
+		glog.V(4).Infof("error reading nbds_max %v", maxNbdsErr)
+		return "", false
+	}
+
+	for i := 0; i < maxNbds; i++ {
+		nbdPath := basePath + strconv.Itoa(i)
+		_, err := os.Lstat(nbdPath)
+		if err != nil {
+			glog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err)
+			continue
+		}
+		pidBytes, err := ioutil.ReadFile(path.Join(nbdPath, "pid"))
+		if err != nil {
+			glog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err)
+			continue
+		}
+		cmdlineFileName := path.Join("/proc", strings.TrimSpace(string(pidBytes)), "cmdline")
+		rawCmdline, err := ioutil.ReadFile(cmdlineFileName)
+		if err != nil {
+			glog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err)
+			continue
+		}
+		cmdlineArgs := strings.FieldsFunc(string(rawCmdline), func(r rune) bool {
+			return r == '\u0000'
+		})
+		// Check if this process is mapping a rbd device.
+		// Only accepted pattern of cmdline is from execRbdMap:
+		// rbd-nbd map pool/image ...
+		if len(cmdlineArgs) < 3 || cmdlineArgs[0] != "rbd-nbd" || cmdlineArgs[1] != "map" {
+			glog.V(4).Infof("nbd device %s is not used by rbd", nbdPath)
+			continue
+		}
+		if cmdlineArgs[2] != imgPath {
+			glog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s",
+				nbdPath, imgPath, cmdlineArgs[2])
+			continue
+		}
+		devicePath := path.Join("/dev", "nbd"+strconv.Itoa(i))
+		if _, err := os.Lstat(devicePath); err != nil {
+			glog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err)
+			continue
+		}
 		return devicePath, true
 	}
-		if i == maxRetries-1 {
-			break
-		}
-		time.Sleep(time.Second)
-	}
 	return "", false
 }
 
-// make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/pool-image-image
+// Stat a path, if it doesn't exist, retry maxRetries times.
+func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
+	for i := 0; i < maxRetries; i++ {
+		if i != 0 {
+			time.Sleep(time.Second)
+		}
+		if useNbdDriver {
+			if devicePath, found := getNbdDevFromImageAndPool(pool, image); found {
+				return devicePath, true
+			}
+		} else {
+			if devicePath, found := getRbdDevFromImageAndPool(pool, image); found {
+				return devicePath, true
+			}
+		}
+	}
+	return "", false
+}
+
+// Execute command to map a rbd device for mounter.
+// rbdCmd is driver dependent and either "rbd" or "rbd-nbd".
+func execRbdMap(b rbdMounter, rbdCmd string, mon string) ([]byte, error) {
+	// Commandline: rbdCmd map imgPath ...
+	// do not change this format - some tools like rbd-nbd are strict about it.
+	imgPath := fmt.Sprintf("%s/%s", b.Pool, b.Image)
+	if b.Secret != "" {
+		return b.exec.Run(rbdCmd,
+			"map", imgPath, "--id", b.Id, "-m", mon, "--key="+b.Secret)
+	} else {
+		return b.exec.Run(rbdCmd,
+			"map", imgPath, "--id", b.Id, "-m", mon, "-k", b.Keyring)
+	}
+}
+
+// Check if rbd-nbd tools are installed.
+func checkRbdNbdTools(e mount.Exec) bool {
+	_, err := e.Run("modprobe", "nbd")
+	if err != nil {
+		glog.V(5).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
+		return false
+	}
+	if _, err := e.Run("rbd-nbd", "--version"); err != nil {
+		glog.V(5).Infof("rbd-nbd: getting rbd-nbd version failed with error %v", err)
+		return false
+	}
+	glog.V(3).Infof("rbd-nbd tools were found.")
+	return true
+}
+
+// Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/pool-image-image.
 func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
-	// Backward compatibility for the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/pool-image-image
+	// Backward compatibility for the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/pool-image-image.
 	deprecatedDir := path.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
 	info, err := os.Stat(deprecatedDir)
 	if err == nil && info.IsDir() {
@@ -125,7 +258,7 @@ func makePDNameInternal(host volume.VolumeHost, pool string, image string) strin
 	return path.Join(host.GetPluginDir(rbdPluginName), mount.MountsInGlobalPDPath, pool+"-image-"+image)
 }
 
-// make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/volumeDevices/pool-image-image
+// Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/volumeDevices/pool-image-image.
 func makeVDPDNameInternal(host volume.VolumeHost, pool string, image string) string {
 	return path.Join(host.GetVolumeDevicePluginDir(rbdPluginName), pool+"-image-"+image)
 }
@@ -156,9 +289,9 @@ func rbdErrors(runErr, resultErr error) error {
 // '--mon_host` parameter (comma, semi-colon, or white-space delimited monitor
 // addresses) and send it to kernel rbd/libceph modules, which can accept
 // comma-seprated list of monitor addresses (e.g. ip1[:port1][,ip2[:port2]...])
-// in theirs first version in linux (see
+// in their first version in linux (see
 // https://github.com/torvalds/linux/blob/602adf400201636e95c3fed9f31fba54a3d7e844/net/ceph/ceph_common.c#L239).
-// Also, libceph module choose monitor randomly, so we can simply pass all
+// Also, libceph module chooses monitor randomly, so we can simply pass all
 // addresses without randomization (see
 // https://github.com/torvalds/linux/blob/602adf400201636e95c3fed9f31fba54a3d7e844/net/ceph/mon_client.c#L132).
 func (util *RBDUtil) kernelRBDMonitorsOpt(mons []string) string {
@@ -184,17 +317,17 @@ func (util *RBDUtil) rbdUnlock(b rbdMounter) error {
 		b.adminSecret = b.Secret
 	}
 
-	// construct lock id using host name and a magic prefix
+	// Construct lock id using host name and a magic prefix.
 	lock_id := kubeLockMagic + node.GetHostname("")
 
 	mon := util.kernelRBDMonitorsOpt(b.Mon)
 
-	// get the locker name, something like "client.1234"
+	// Get the locker name, something like "client.1234".
 	args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}
 	args = append(args, secret_opt...)
 	cmd, err = b.exec.Run("rbd", args...)
 	output = string(cmd)
-	glog.Infof("lock list output %q", output)
+	glog.V(4).Infof("lock list output %q", output)
 	if err != nil {
 		return err
 	}
@@ -206,7 +339,7 @@ func (util *RBDUtil) rbdUnlock(b rbdMounter) error {
 		}
 	}
 
-	// remove a lock if found: rbd lock remove
+	// Remove a lock if found: rbd lock remove.
 	if len(locker) > 0 {
 		args := []string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}
 		args = append(args, secret_opt...)
@@ -221,7 +354,6 @@ func (util *RBDUtil) rbdUnlock(b rbdMounter) error {
 
 // AttachDisk attaches the disk on the node.
 func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) {
-	var err error
 	var output []byte
 
 	globalPDPath := util.MakeGlobalPDName(*b.rbd)
@@ -233,13 +365,20 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) {
 		}
 	}
 
-	devicePath, found := waitForPath(b.Pool, b.Image, 1)
+	// Evalute whether this device was mapped with rbd.
-	if !found {
+	devicePath, mapped := waitForPath(b.Pool, b.Image, 1 /*maxRetries*/, false /*useNbdDriver*/)
-		_, err = b.exec.Run("modprobe", "rbd")
+
-		if err != nil {
+	// If rbd-nbd tools are found, we will fallback to it should the default krbd driver fail.
-			glog.Warningf("rbd: failed to load rbd kernel module:%v", err)
+	nbdToolsFound := false
+
+	if !mapped {
+		nbdToolsFound := checkRbdNbdTools(b.exec)
+		if nbdToolsFound {
+			devicePath, mapped = waitForPath(b.Pool, b.Image, 1 /*maxRetries*/, true /*useNbdDriver*/)
+		}
 	}
 
+	if !mapped {
 		// Currently, we don't acquire advisory lock on image, but for backward
 		// compatibility, we need to check if the image is being used by nodes running old kubelet.
 		// osd_client_watch_timeout defaults to 30 seconds, if the watcher stays active longer than 30 seconds,
@@ -256,30 +395,42 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) {
 			}
 			return !used, nil
 		})
-		// return error if rbd image has not become available for the specified timeout
+		// Return error if rbd image has not become available for the specified timeout.
 		if err == wait.ErrWaitTimeout {
 			return "", fmt.Errorf("rbd image %s/%s is still being used", b.Pool, b.Image)
 		}
-		// return error if any other errors were encountered during wating for the image to becme avialble
+		// Return error if any other errors were encountered during wating for the image to become available.
 		if err != nil {
 			return "", err
 		}
 
 		mon := util.kernelRBDMonitorsOpt(b.Mon)
 		glog.V(1).Infof("rbd: map mon %s", mon)
-		if b.Secret != "" {
+
-			output, err = b.exec.Run("rbd",
+		_, err = b.exec.Run("modprobe", "rbd")
-				"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key="+b.Secret)
-		} else {
-			output, err = b.exec.Run("rbd",
-				"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring)
-		}
 		if err != nil {
+			glog.Warningf("rbd: failed to load rbd kernel module:%v", err)
+		}
+		output, err = execRbdMap(b, "rbd", mon)
+		if err != nil {
+			if !nbdToolsFound {
 				glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
 				return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
 			}
-		devicePath, found = waitForPath(b.Pool, b.Image, 10)
+			glog.V(3).Infof("rbd: map failed with %v, %s. Retrying with rbd-nbd", err, string(output))
-		if !found {
+			errList := []error{err}
+			outputList := output
+			output, err = execRbdMap(b, "rbd-nbd", mon)
+			if err != nil {
+				errList = append(errList, err)
+				outputList = append(outputList, output...)
+				return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", errors.NewAggregate(errList), string(outputList))
+			}
+			devicePath, mapped = waitForPath(b.Pool, b.Image, 10 /*maxRetries*/, true /*useNbdDrive*/)
+		} else {
+			devicePath, mapped = waitForPath(b.Pool, b.Image, 10 /*maxRetries*/, false /*useNbdDriver*/)
+		}
+		if !mapped {
 			return "", fmt.Errorf("Could not map image %s/%s, Timeout after 10s", b.Pool, b.Image)
 		}
 	}
@@ -293,9 +444,21 @@ func (util *RBDUtil) DetachDisk(plugin *rbdPlugin, deviceMountPath string, devic
 	if len(device) == 0 {
 		return fmt.Errorf("DetachDisk failed , device is empty")
 	}
-	// rbd unmap
+
 	exec := plugin.host.GetExec(plugin.GetPluginName())
-	output, err := exec.Run("rbd", "unmap", device)
+
+	var rbdCmd string
+
+	// Unlike map, we cannot fallthrough for unmap
+	// the tool to unmap is based on device type
+	if strings.HasPrefix(device, "/dev/nbd") {
+		rbdCmd = "rbd-nbd"
+	} else {
+		rbdCmd = "rbd"
+	}
+
+	// rbd unmap
+	output, err := exec.Run(rbdCmd, "unmap", device)
 	if err != nil {
 		return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %v", device, err, output))
 	}
@@ -329,7 +492,7 @@ func (util *RBDUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error
 		glog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
 		return nil
 	}
-	// If we arrive here, device is no longer used, see if need to logout the target
+	// If we arrive here, device is no longer used, see if we need to logout of the target
 	device, err := getBlockVolumeDevice(mapPath)
 	if err != nil {
 		return err
@@ -338,9 +501,23 @@ func (util *RBDUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error
 	if len(device) == 0 {
 		return fmt.Errorf("DetachDisk failed , device is empty")
 	}
-	// rbd unmap
+
 	exec := disk.plugin.host.GetExec(disk.plugin.GetPluginName())
-	output, err := exec.Run("rbd", "unmap", device)
+
+	var rbdCmd string
+
+	// Unlike map, we cannot fallthrough here.
+	// Any nbd device must be unmapped by rbd-nbd
+	if strings.HasPrefix(device, "/dev/nbd") {
+		rbdCmd = "rbd-nbd"
+		glog.V(4).Infof("rbd: using rbd-nbd for unmap function")
+	} else {
+		rbdCmd = "rbd"
+		glog.V(4).Infof("rbd: using rbd for unmap function")
+	}
+
+	// rbd unmap
+	output, err := exec.Run(rbdCmd, "unmap", device)
 	if err != nil {
 		return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %s", device, err, string(output)))
 	}
@@ -371,9 +548,9 @@ func (util *RBDUtil) cleanOldRBDFile(plugin *rbdPlugin, rbdFile string) error {
 		glog.Errorf("failed to load rbd info from %s: %v", rbdFile, err)
 		return err
 	}
-	// remove rbd lock if found
+	// Remove rbd lock if found.
-	// the disk is not attached to this node anymore, so the lock on image
+	// The disk is not attached to this node anymore, so the lock on image
-	// for this node can be removed safely
+	// for this node can be removed safely.
 	err = util.rbdUnlock(*mounter)
 	if err == nil {
 		os.Remove(rbdFile)
@@ -385,7 +562,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVo
 	var output []byte
 	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
 	volSizeBytes := capacity.Value()
-	// convert to MB that rbd defaults on
+	// Convert to MB that rbd defaults on.
 	sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
 	volSz := fmt.Sprintf("%d", sz)
 	mon := util.kernelRBDMonitorsOpt(p.Mon)
@@ -396,8 +573,8 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVo
 	}
 	args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
 	if p.rbdMounter.imageFormat == rbdImageFormat2 {
-		// if no image features is provided, it results in empty string
+		// If no image features is provided, it results in empty string
-		// which disable all RBD image format 2 features as we expected
+		// which disable all RBD image format 2 features as expected.
 		features := strings.Join(p.rbdMounter.imageFeatures, ",")
 		args = append(args, "--image-feature", features)
 	}
@@ -425,7 +602,7 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
 		glog.Info("rbd is still being used ", p.rbdMounter.Image)
 		return fmt.Errorf("rbd image %s/%s is still being used, rbd output: %v", p.rbdMounter.Pool, p.rbdMounter.Image, rbdOutput)
 	}
-	// rbd rm
+	// rbd rm.
 	mon := util.kernelRBDMonitorsOpt(p.rbdMounter.Mon)
 	glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
 	output, err = p.exec.Run("rbd",
@@ -438,17 +615,17 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
 	return fmt.Errorf("error %v, rbd output: %v", err, string(output))
 }
 
-// ExpandImage runs rbd resize command to resize the specified image
+// ExpandImage runs rbd resize command to resize the specified image.
 func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
 	var output []byte
 	var err error
 	volSizeBytes := newSize.Value()
-	// convert to MB that rbd defaults on
+	// Convert to MB that rbd defaults on.
 	sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
 	newVolSz := fmt.Sprintf("%d", sz)
 	newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
 
-	// check the current size of rbd image, if equals to or greater that the new request size, do nothing
+	// Check the current size of rbd image, if equals to or greater that the new request size, do nothing.
 	curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
 	if infoErr != nil {
 		return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
@@ -457,7 +634,7 @@ func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resourc
 		return newSizeQuant, nil
 	}
 
-	// rbd resize
+	// rbd resize.
 	mon := util.kernelRBDMonitorsOpt(rbdExpander.rbdMounter.Mon)
 	glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
 	output, err = rbdExpander.exec.Run("rbd",
@@ -470,7 +647,7 @@ func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resourc
 	return oldSize, err
 }
 
-// rbdInfo runs `rbd info` command to get the current image size in MB
+// rbdInfo runs `rbd info` command to get the current image size in MB.
 func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
 	var err error
 	var output string
@@ -510,7 +687,7 @@ func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
 	if err, ok := err.(*exec.Error); ok {
 		if err.Err == exec.ErrNotFound {
 			glog.Errorf("rbd cmd not found")
-			// fail fast if command not found
+			// fail fast if rbd command is not found.
 			return 0, err
 		}
 	}
@@ -524,7 +701,7 @@ func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
 		return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, output)
 	}
 
-	// get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects`
+	// Get the size value string, just between `size ` and ` MB in`, such as `size 1024 MB in 256 objects`.
 	sizeIndex := strings.Index(output, imageSizeStr)
 	divIndex := strings.Index(output, sizeDivStr)
 	if sizeIndex == -1 || divIndex == -1 || divIndex <= sizeIndex+5 {