diff --git a/pkg/volume/rbd/BUILD b/pkg/volume/rbd/BUILD index 32b6efcfab8..f906a9dea4d 100644 --- a/pkg/volume/rbd/BUILD +++ b/pkg/volume/rbd/BUILD @@ -29,7 +29,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 4b58a10bb27..4aa87d9db89 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -33,7 +33,6 @@ import ( "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) var ( @@ -42,12 +41,11 @@ var ( // This is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&rbdPlugin{nil, exec.New()}} + return []volume.VolumePlugin{&rbdPlugin{nil}} } type rbdPlugin struct { host volume.VolumeHost - exe exec.Interface } var _ volume.VolumePlugin = &rbdPlugin{} @@ -131,7 +129,7 @@ func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.Vol } // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), secret) + return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret) } func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSource, bool) { @@ -144,7 +142,7 @@ func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSou } } -func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, secret string) (volume.Mounter, error) { +func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret string) (volume.Mounter, error) { source, readOnly := plugin.getRBDVolumeSource(spec) pool := source.RBDPool id := source.RadosUser @@ -158,7 +156,8 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, Pool: pool, ReadOnly: readOnly, manager: manager, - mounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host), + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)), }, @@ -173,17 +172,18 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) + return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) { +func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) { return &rbdUnmounter{ rbdMounter: &rbdMounter{ rbd: &rbd{ podUID: podUID, volName: volName, manager: manager, - mounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host), + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)), }, @@ -246,6 +246,8 @@ func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret str Pool: spec.PersistentVolume.Spec.RBD.RBDPool, manager: manager, plugin: plugin, + mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())}, + exec: plugin.host.GetExec(plugin.GetPluginName()), }, Mon: spec.PersistentVolume.Spec.RBD.CephMonitors, adminId: admin, @@ -263,6 +265,8 @@ func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, ma rbd: &rbd{ manager: manager, plugin: plugin, + mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())}, + exec: plugin.host.GetExec(plugin.GetPluginName()), }, }, options: options, @@ -402,6 +406,7 @@ type rbd struct { ReadOnly bool plugin *rbdPlugin mounter *mount.SafeFormatAndMount + exec mount.Exec // Utility interface that provides API calls to the provider to attach/detach disks. manager diskManager volume.MetricsProvider @@ -480,11 +485,6 @@ func (c *rbdUnmounter) TearDownAt(dir string) error { return diskTearDown(c.manager, *c, dir, c.mounter) } -func (plugin *rbdPlugin) execCommand(command string, args []string) ([]byte, error) { - cmd := plugin.exe.Command(command, args...) - return cmd.CombinedOutput() -} - func getVolumeSource( spec *volume.Spec) (*v1.RBDVolumeSource, bool, error) { if spec.Volume != nil && spec.Volume.RBD != nil { diff --git a/pkg/volume/rbd/rbd_test.go b/pkg/volume/rbd/rbd_test.go index 6ef38de89ec..a756188bb88 100644 --- a/pkg/volume/rbd/rbd_test.go +++ b/pkg/volume/rbd/rbd_test.go @@ -112,7 +112,8 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { } fdm := NewFakeDiskManager() defer fdm.Cleanup() - mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, "secrets") + exec := mount.NewFakeExec(nil) + mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, exec, "secrets") if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -137,7 +138,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { } } - unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}) + unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}, exec) if err != nil { t.Errorf("Failed to make a new Unmounter: %v", err) } diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index 001054e903a..08b8af5407b 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -38,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/volume" - "k8s.io/utils/exec" ) const ( @@ -58,15 +57,24 @@ func getDevFromImageAndPool(pool, image string) (string, bool) { // https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c name := f.Name() // first match pool, then match name - po := path.Join(sys_path, name, "pool") - img := path.Join(sys_path, name, "name") - exe := exec.New() - out, err := exe.Command("cat", po, img).CombinedOutput() + poolFile := path.Join(sys_path, name, "pool") + poolBytes, err := ioutil.ReadFile(poolFile) if err != nil { + glog.V(4).Infof("Error reading %s: %v", poolFile, err) continue } - matched, err := regexp.MatchString("^"+pool+"\n"+image+"\n$", string(out)) - if err != nil || !matched { + if strings.TrimSpace(string(poolBytes)) != pool { + glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes)) + continue + } + imgFile := path.Join(sys_path, name, "name") + imgBytes, err := ioutil.ReadFile(imgFile) + if err != nil { + glog.V(4).Infof("Error reading %s: %v", imgFile, err) + continue + } + if strings.TrimSpace(string(imgBytes)) != image { + glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes)) continue } // found a match, check if device exists @@ -142,8 +150,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { // for fencing, check if lock already held for this host // this edge case happens if host crashes in the middle of acquiring lock and mounting rbd // for defencing, get the locker name, something like "client.1234" - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) output = string(cmd) glog.Infof("lock list output %q", output) if err != nil { @@ -166,8 +175,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { if len(v) > 0 { lockInfo := strings.Split(v[0], " ") if len(lockInfo) > 2 { - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd)) } } @@ -175,8 +185,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { } // hold a lock: rbd lock add - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) } else { // defencing, find locker name ind := strings.LastIndex(output, lock_id) - 1 @@ -187,8 +198,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { } } // remove a lock: rbd lock remove - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) } if err == nil { @@ -268,8 +280,7 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { devicePath, found := waitForPath(b.Pool, b.Image, 1) if !found { - // modprobe - _, err = b.plugin.execCommand("modprobe", []string{"rbd"}) + _, err = b.exec.Run("modprobe", "rbd") if err != nil { glog.Warningf("rbd: failed to load rbd kernel module:%v", err) } @@ -294,11 +305,11 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { mon := b.Mon[i%l] glog.V(1).Infof("rbd: map mon %s", mon) if b.Secret != "" { - output, err = b.plugin.execCommand("rbd", - []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key=" + b.Secret}) + output, err = b.exec.Run("rbd", + "map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key="+b.Secret) } else { - output, err = b.plugin.execCommand("rbd", - []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring}) + output, err = b.exec.Run("rbd", + "map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring) } if err == nil { break @@ -332,7 +343,7 @@ func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error { // if device is no longer used, see if can unmap if cnt <= 1 { // rbd unmap - _, err = c.plugin.execCommand("rbd", []string{"unmap", device}) + _, err = c.exec.Run("rbd", "unmap", device) if err != nil { return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s:Error: %v", device, err)) } @@ -374,7 +385,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource features := strings.Join(p.rbdMounter.imageFeatures, ",") args = append(args, "--image-feature", features) } - output, err = p.rbdMounter.plugin.execCommand("rbd", args) + output, err = p.exec.Run("rbd", args...) if err == nil { break } else { @@ -411,8 +422,8 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error { for i := start; i < start+l; i++ { mon := p.rbdMounter.Mon[i%l] glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) - output, err = p.plugin.execCommand("rbd", - []string{"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret}) + output, err = p.exec.Run("rbd", + "rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key="+p.rbdMounter.adminSecret) if err == nil { return nil } else { @@ -437,8 +448,8 @@ func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, error) { // Watchers: // watcher=10.16.153.105:0/710245699 client.14163 cookie=1 glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, b.adminId, b.adminSecret) - cmd, err = b.plugin.execCommand("rbd", - []string{"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key=" + b.adminSecret}) + cmd, err = b.exec.Run("rbd", + "status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key="+b.adminSecret) output = string(cmd) if err != nil {