mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 15:05:27 +00:00
rbd: Use VolumeHost.GetExec() to execute stuff in volume plugins
This commit is contained in:
Jan Safranek
parent
07dea6b447
commit
73b101c14b
@ -29,7 +29,6 @@ go_library(
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -33,7 +33,6 @@ import (
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
volutil "k8s.io/kubernetes/pkg/volume/util"
|
||||
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
|
||||
"k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -42,12 +41,11 @@ var (
|
||||
|
||||
// This is the primary entrypoint for volume plugins.
|
||||
func ProbeVolumePlugins() []volume.VolumePlugin {
|
||||
return []volume.VolumePlugin{&rbdPlugin{nil, exec.New()}}
|
||||
return []volume.VolumePlugin{&rbdPlugin{nil}}
|
||||
}
|
||||
|
||||
type rbdPlugin struct {
|
||||
host volume.VolumeHost
|
||||
exe exec.Interface
|
||||
}
|
||||
|
||||
var _ volume.VolumePlugin = &rbdPlugin{}
|
||||
@ -131,7 +129,7 @@ func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.Vol
|
||||
}
|
||||
|
||||
// Inject real implementations here, test through the internal function.
|
||||
return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), secret)
|
||||
return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSource, bool) {
|
||||
@ -144,7 +142,7 @@ func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSou
|
||||
}
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, secret string) (volume.Mounter, error) {
|
||||
func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret string) (volume.Mounter, error) {
|
||||
source, readOnly := plugin.getRBDVolumeSource(spec)
|
||||
pool := source.RBDPool
|
||||
id := source.RadosUser
|
||||
@ -158,7 +156,8 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID,
|
||||
Pool: pool,
|
||||
ReadOnly: readOnly,
|
||||
manager: manager,
|
||||
mounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
|
||||
mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
|
||||
exec: exec,
|
||||
plugin: plugin,
|
||||
MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
|
||||
},
|
||||
@ -173,17 +172,18 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID,
|
||||
|
||||
func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
|
||||
// Inject real implementations here, test through the internal function.
|
||||
return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
|
||||
return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) {
|
||||
func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) {
|
||||
return &rbdUnmounter{
|
||||
rbdMounter: &rbdMounter{
|
||||
rbd: &rbd{
|
||||
podUID: podUID,
|
||||
volName: volName,
|
||||
manager: manager,
|
||||
mounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
|
||||
mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
|
||||
exec: exec,
|
||||
plugin: plugin,
|
||||
MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
|
||||
},
|
||||
@ -246,6 +246,8 @@ func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret str
|
||||
Pool: spec.PersistentVolume.Spec.RBD.RBDPool,
|
||||
manager: manager,
|
||||
plugin: plugin,
|
||||
mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
|
||||
exec: plugin.host.GetExec(plugin.GetPluginName()),
|
||||
},
|
||||
Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
|
||||
adminId: admin,
|
||||
@ -263,6 +265,8 @@ func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, ma
|
||||
rbd: &rbd{
|
||||
manager: manager,
|
||||
plugin: plugin,
|
||||
mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
|
||||
exec: plugin.host.GetExec(plugin.GetPluginName()),
|
||||
},
|
||||
},
|
||||
options: options,
|
||||
@ -402,6 +406,7 @@ type rbd struct {
|
||||
ReadOnly bool
|
||||
plugin *rbdPlugin
|
||||
mounter *mount.SafeFormatAndMount
|
||||
exec mount.Exec
|
||||
// Utility interface that provides API calls to the provider to attach/detach disks.
|
||||
manager diskManager
|
||||
volume.MetricsProvider
|
||||
@ -480,11 +485,6 @@ func (c *rbdUnmounter) TearDownAt(dir string) error {
|
||||
return diskTearDown(c.manager, *c, dir, c.mounter)
|
||||
}
|
||||
|
||||
func (plugin *rbdPlugin) execCommand(command string, args []string) ([]byte, error) {
|
||||
cmd := plugin.exe.Command(command, args...)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func getVolumeSource(
|
||||
spec *volume.Spec) (*v1.RBDVolumeSource, bool, error) {
|
||||
if spec.Volume != nil && spec.Volume.RBD != nil {
|
||||
|
||||
@ -112,7 +112,8 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) {
|
||||
}
|
||||
fdm := NewFakeDiskManager()
|
||||
defer fdm.Cleanup()
|
||||
mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, "secrets")
|
||||
exec := mount.NewFakeExec(nil)
|
||||
mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, exec, "secrets")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to make a new Mounter: %v", err)
|
||||
}
|
||||
@ -137,7 +138,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) {
|
||||
}
|
||||
}
|
||||
|
||||
unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{})
|
||||
unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}, exec)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to make a new Unmounter: %v", err)
|
||||
}
|
||||
|
||||
@ -38,7 +38,6 @@ import (
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
"k8s.io/kubernetes/pkg/util/node"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
"k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -58,15 +57,24 @@ func getDevFromImageAndPool(pool, image string) (string, bool) {
|
||||
// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
|
||||
name := f.Name()
|
||||
// first match pool, then match name
|
||||
po := path.Join(sys_path, name, "pool")
|
||||
img := path.Join(sys_path, name, "name")
|
||||
exe := exec.New()
|
||||
out, err := exe.Command("cat", po, img).CombinedOutput()
|
||||
poolFile := path.Join(sys_path, name, "pool")
|
||||
poolBytes, err := ioutil.ReadFile(poolFile)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("Error reading %s: %v", poolFile, err)
|
||||
continue
|
||||
}
|
||||
matched, err := regexp.MatchString("^"+pool+"\n"+image+"\n$", string(out))
|
||||
if err != nil || !matched {
|
||||
if strings.TrimSpace(string(poolBytes)) != pool {
|
||||
glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes))
|
||||
continue
|
||||
}
|
||||
imgFile := path.Join(sys_path, name, "name")
|
||||
imgBytes, err := ioutil.ReadFile(imgFile)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("Error reading %s: %v", imgFile, err)
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(string(imgBytes)) != image {
|
||||
glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes))
|
||||
continue
|
||||
}
|
||||
// found a match, check if device exists
|
||||
@ -142,8 +150,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
|
||||
// for fencing, check if lock already held for this host
|
||||
// this edge case happens if host crashes in the middle of acquiring lock and mounting rbd
|
||||
// for defencing, get the locker name, something like "client.1234"
|
||||
cmd, err = b.plugin.execCommand("rbd",
|
||||
append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
|
||||
args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}
|
||||
args = append(args, secret_opt...)
|
||||
cmd, err = b.exec.Run("rbd", args...)
|
||||
output = string(cmd)
|
||||
glog.Infof("lock list output %q", output)
|
||||
if err != nil {
|
||||
@ -166,8 +175,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
|
||||
if len(v) > 0 {
|
||||
lockInfo := strings.Split(v[0], " ")
|
||||
if len(lockInfo) > 2 {
|
||||
cmd, err = b.plugin.execCommand("rbd",
|
||||
append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
|
||||
args := []string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}
|
||||
args = append(args, secret_opt...)
|
||||
cmd, err = b.exec.Run("rbd", args...)
|
||||
glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd))
|
||||
}
|
||||
}
|
||||
@ -175,8 +185,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
|
||||
}
|
||||
|
||||
// hold a lock: rbd lock add
|
||||
cmd, err = b.plugin.execCommand("rbd",
|
||||
append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
|
||||
args := []string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}
|
||||
args = append(args, secret_opt...)
|
||||
cmd, err = b.exec.Run("rbd", args...)
|
||||
} else {
|
||||
// defencing, find locker name
|
||||
ind := strings.LastIndex(output, lock_id) - 1
|
||||
@ -187,8 +198,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
|
||||
}
|
||||
}
|
||||
// remove a lock: rbd lock remove
|
||||
cmd, err = b.plugin.execCommand("rbd",
|
||||
append([]string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
|
||||
args := []string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}
|
||||
args = append(args, secret_opt...)
|
||||
cmd, err = b.exec.Run("rbd", args...)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
@ -268,8 +280,7 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
|
||||
|
||||
devicePath, found := waitForPath(b.Pool, b.Image, 1)
|
||||
if !found {
|
||||
// modprobe
|
||||
_, err = b.plugin.execCommand("modprobe", []string{"rbd"})
|
||||
_, err = b.exec.Run("modprobe", "rbd")
|
||||
if err != nil {
|
||||
glog.Warningf("rbd: failed to load rbd kernel module:%v", err)
|
||||
}
|
||||
@ -294,11 +305,11 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
|
||||
mon := b.Mon[i%l]
|
||||
glog.V(1).Infof("rbd: map mon %s", mon)
|
||||
if b.Secret != "" {
|
||||
output, err = b.plugin.execCommand("rbd",
|
||||
[]string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key=" + b.Secret})
|
||||
output, err = b.exec.Run("rbd",
|
||||
"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key="+b.Secret)
|
||||
} else {
|
||||
output, err = b.plugin.execCommand("rbd",
|
||||
[]string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring})
|
||||
output, err = b.exec.Run("rbd",
|
||||
"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring)
|
||||
}
|
||||
if err == nil {
|
||||
break
|
||||
@ -332,7 +343,7 @@ func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error {
|
||||
// if device is no longer used, see if can unmap
|
||||
if cnt <= 1 {
|
||||
// rbd unmap
|
||||
_, err = c.plugin.execCommand("rbd", []string{"unmap", device})
|
||||
_, err = c.exec.Run("rbd", "unmap", device)
|
||||
if err != nil {
|
||||
return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s:Error: %v", device, err))
|
||||
}
|
||||
@ -374,7 +385,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource
|
||||
features := strings.Join(p.rbdMounter.imageFeatures, ",")
|
||||
args = append(args, "--image-feature", features)
|
||||
}
|
||||
output, err = p.rbdMounter.plugin.execCommand("rbd", args)
|
||||
output, err = p.exec.Run("rbd", args...)
|
||||
if err == nil {
|
||||
break
|
||||
} else {
|
||||
@ -411,8 +422,8 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
|
||||
for i := start; i < start+l; i++ {
|
||||
mon := p.rbdMounter.Mon[i%l]
|
||||
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
|
||||
output, err = p.plugin.execCommand("rbd",
|
||||
[]string{"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret})
|
||||
output, err = p.exec.Run("rbd",
|
||||
"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key="+p.rbdMounter.adminSecret)
|
||||
if err == nil {
|
||||
return nil
|
||||
} else {
|
||||
@ -437,8 +448,8 @@ func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, error) {
|
||||
// Watchers:
|
||||
// watcher=10.16.153.105:0/710245699 client.14163 cookie=1
|
||||
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, b.adminId, b.adminSecret)
|
||||
cmd, err = b.plugin.execCommand("rbd",
|
||||
[]string{"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key=" + b.adminSecret})
|
||||
cmd, err = b.exec.Run("rbd",
|
||||
"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key="+b.adminSecret)
|
||||
output = string(cmd)
|
||||
|
||||
if err != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user