From 1a91403e435f4b8e425e864890653747528bf8d8 Mon Sep 17 00:00:00 2001 From: Simon Croome Date: Tue, 6 Feb 2018 13:28:36 +0000 Subject: [PATCH] StorageOS support for containerized kubelet and mount options This change queries the StorageOS API to retrieve the volume device dir when a volume is attached, rather than relying on the default /var/lib/storageos/volumes directory. This allows all or some nodes to have kubelet running in a container with the StorageOS volume path mounted in, which is required for Azure ACS support. Volume mount options are also supported. --- pkg/volume/storageos/storageos.go | 22 ++++++++++------ pkg/volume/storageos/storageos_test.go | 9 +++++++ pkg/volume/storageos/storageos_util.go | 28 +++++++++++++++++++-- pkg/volume/storageos/storageos_util_test.go | 5 +++- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index 532ccd28987..5f9c2aa2c27 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -54,7 +54,7 @@ var _ volume.ProvisionableVolumePlugin = &storageosPlugin{} const ( storageosPluginName = "kubernetes.io/storageos" - storageosDevicePath = "/var/lib/storageos/volumes" + defaultDeviceDir = "/var/lib/storageos/volumes" defaultAPIAddress = "tcp://localhost:5705" defaultAPIUser = "storageos" defaultAPIPassword = "storageos" @@ -136,8 +136,8 @@ func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, volNamespace, volName, spec.Name(), plugin.host)), }, - devicePath: storageosDevicePath, - diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + mountOptions: volume.MountOptionFromSpec(spec), }, nil } @@ -249,7 +249,7 @@ func (plugin *storageosPlugin) ConstructVolumeSpec(volumeName, mountPath string) } func (plugin *storageosPlugin) SupportsMountOption() bool { - return false + return true } func (plugin *storageosPlugin) SupportsBulkVolumeVerification() bool { @@ -286,6 +286,8 @@ type storageosManager interface { UnmountVolume(unounter *storageosUnmounter) error // Deletes the storageos volume. All data will be lost. DeleteVolume(deleter *storageosDeleter) error + // Gets the node's device path. + DeviceDir(mounter *storageosMounter) string } // storageos volumes represent a bare host directory mount of an StorageOS export. @@ -312,9 +314,13 @@ type storageos struct { type storageosMounter struct { *storageos - devicePath string + + // The directory containing the StorageOS devices + deviceDir string + // Interface used to mount the file or block device - diskMounter *mount.SafeFormatAndMount + diskMounter *mount.SafeFormatAndMount + mountOptions []string } var _ volume.Mounter = &storageosMounter{} @@ -383,11 +389,12 @@ func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { if b.readOnly { options = append(options, "ro") } + mountOptions := volume.JoinMountOptions(b.mountOptions, options) globalPDPath := makeGlobalPDName(b.plugin.host, b.pvName, b.volNamespace, b.volName) glog.V(4).Infof("Attempting to bind mount to pod volume at %s", dir) - err = b.mounter.Mount(globalPDPath, dir, "", options) + err = b.mounter.Mount(globalPDPath, dir, "", mountOptions) if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { @@ -636,6 +643,7 @@ func (c *storageosProvisioner) Provision() (*v1.PersistentVolume, error) { }, }, }, + MountOptions: c.options.MountOptions, }, } if len(c.options.PVC.Spec.AccessModes) == 0 { diff --git a/pkg/volume/storageos/storageos_test.go b/pkg/volume/storageos/storageos_test.go index a77f2c72790..9d03ab235ac 100644 --- a/pkg/volume/storageos/storageos_test.go +++ b/pkg/volume/storageos/storageos_test.go @@ -131,6 +131,10 @@ func (fake *fakePDManager) DeleteVolume(d *storageosDeleter) error { return nil } +func (fake *fakePDManager) DeviceDir(mounter *storageosMounter) string { + return defaultDeviceDir +} + func TestPlugin(t *testing.T) { tmpDir, err := utiltesting.MkTmpdir("storageos_test") if err != nil { @@ -249,6 +253,7 @@ func TestPlugin(t *testing.T) { // Test Provisioner fakeManager = &fakePDManager{} + mountOptions := []string{"sync", "noatime"} options := volume.VolumeOptions{ PVC: volumetest.CreateTestPVC("100Mi", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}), // PVName: "test-volume-name", @@ -257,6 +262,7 @@ func TestPlugin(t *testing.T) { "VolumeNamespace": "test-volume-namespace", "adminSecretName": secretName, }, + MountOptions: mountOptions, } provisioner, err := plug.(*storageosPlugin).newProvisionerInternal(options, fakeManager) if err != nil { @@ -282,6 +288,9 @@ func TestPlugin(t *testing.T) { if persistentSpec.Spec.PersistentVolumeSource.StorageOS.FSType != "ext2" { t.Errorf("Provision() returned unexpected volume FSType: %s", persistentSpec.Spec.PersistentVolumeSource.StorageOS.FSType) } + if len(persistentSpec.Spec.MountOptions) != 2 { + t.Errorf("Provision() returned unexpected volume mount options: %v", persistentSpec.Spec.MountOptions) + } if persistentSpec.Labels["fakepdmanager"] != "yes" { t.Errorf("Provision() returned unexpected labels: %v", persistentSpec.Labels) } diff --git a/pkg/volume/storageos/storageos_util.go b/pkg/volume/storageos/storageos_util.go index 107ba859013..beac8924087 100644 --- a/pkg/volume/storageos/storageos_util.go +++ b/pkg/volume/storageos/storageos_util.go @@ -69,6 +69,7 @@ type apiImplementer interface { VolumeMount(opts storageostypes.VolumeMountOptions) error VolumeUnmount(opts storageostypes.VolumeUnmountOptions) error VolumeDelete(opt storageostypes.DeleteOptions) error + Controller(ref string) (*storageostypes.Controller, error) } // storageosUtil is the utility structure to interact with the StorageOS API. @@ -148,6 +149,12 @@ func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { return "", err } + // Get the node's device path from the API, falling back to the default if + // not set on the node. + if b.deviceDir == "" { + b.deviceDir = u.DeviceDir(b) + } + vol, err := u.api.Volume(b.volNamespace, b.volName) if err != nil { glog.Warningf("volume retrieve failed for volume %q with namespace %q (%v)", b.volName, b.volNamespace, err) @@ -167,12 +174,13 @@ func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { } } - srcPath := path.Join(b.devicePath, vol.ID) + srcPath := path.Join(b.deviceDir, vol.ID) dt, err := pathDeviceType(srcPath) if err != nil { glog.Warningf("volume source path %q for volume %q not ready (%v)", srcPath, b.volName, err) return "", err } + switch dt { case modeBlock: return srcPath, nil @@ -277,6 +285,22 @@ func (u *storageosUtil) DeleteVolume(d *storageosDeleter) error { return u.api.VolumeDelete(opts) } +// Get the node's device path from the API, falling back to the default if not +// specified. +func (u *storageosUtil) DeviceDir(b *storageosMounter) string { + + ctrl, err := u.api.Controller(b.plugin.host.GetHostName()) + if err != nil { + glog.Warningf("node device path lookup failed: %v", err) + return defaultDeviceDir + } + if ctrl == nil || ctrl.DeviceDir == "" { + glog.Warningf("node device path not set, using default: %s", defaultDeviceDir) + return defaultDeviceDir + } + return ctrl.DeviceDir +} + // pathMode returns the FileMode for a path. func pathDeviceType(path string) (deviceType, error) { fi, err := os.Stat(path) @@ -332,7 +356,7 @@ func getLoopDevice(path string, exec mount.Exec) (string, error) { } func makeLoopDevice(path string, exec mount.Exec) (string, error) { - args := []string{"-f", "--show", path} + args := []string{"-f", "-P", "--show", path} out, err := exec.Run(losetupPath, args...) if err != nil { glog.V(2).Infof("Failed device create command for path %s: %v", path, err) diff --git a/pkg/volume/storageos/storageos_util_test.go b/pkg/volume/storageos/storageos_util_test.go index 3eee51231e7..bff34ca7d01 100644 --- a/pkg/volume/storageos/storageos_util_test.go +++ b/pkg/volume/storageos/storageos_util_test.go @@ -108,6 +108,9 @@ func (f fakeAPI) VolumeUnmount(opts storageostypes.VolumeUnmountOptions) error { func (f fakeAPI) VolumeDelete(opts storageostypes.DeleteOptions) error { return nil } +func (f fakeAPI) Controller(ref string) (*storageostypes.Controller, error) { + return &storageostypes.Controller{}, nil +} func TestCreateVolume(t *testing.T) { @@ -224,7 +227,7 @@ func TestAttachVolume(t *testing.T) { mounter: &mount.FakeMounter{}, plugin: plug.(*storageosPlugin), }, - devicePath: tmpDir, + deviceDir: tmpDir, } if err != nil { t.Errorf("Failed to make a new Mounter: %v", err)