Merge pull request #63011 from NickrenREN/local-plugin-change

Automatic merge from submit-queue (batch tested with PRs 63011, 68089, 67944, 68132). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Support both directory and block device for local volume plugin FileSystem VolumeMode

Support both directory and block device for local volume plugin FileSystem VolumeMode 

xref: [local storage dynamic provisioning design #1914](https://github.com/kubernetes/community/pull/1914)

**What this PR does / why we need it**:

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:
```release-note
Support both directory and block device for local volume plugin FileSystem VolumeMode 
```
This commit is contained in:
Kubernetes Submit Queue 2018-09-04 10:24:36 -07:00 committed by GitHub
commit 9c86087dba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 972 additions and 583 deletions

View File

@ -81795,8 +81795,12 @@
"path" "path"
], ],
"properties": { "properties": {
"fsType": {
"description": "Filesystem type to mount. It applies only when the Path is a block device. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default value is to auto-select a fileystem if unspecified.",
"type": "string"
},
"path": { "path": {
"description": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...). Directories can be represented only by PersistentVolume with VolumeMode=Filesystem. Block devices can be represented only by VolumeMode=Block, which also requires the BlockVolume alpha feature gate to be enabled.", "description": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...).",
"type": "string" "type": "string"
} }
} }

View File

@ -20242,7 +20242,11 @@
"properties": { "properties": {
"path": { "path": {
"type": "string", "type": "string",
"description": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...). Directories can be represented only by PersistentVolume with VolumeMode=Filesystem. Block devices can be represented only by VolumeMode=Block, which also requires the BlockVolume alpha feature gate to be enabled." "description": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...)."
},
"fsType": {
"type": "string",
"description": "Filesystem type to mount. It applies only when the Path is a block device. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default value is to auto-select a fileystem if unspecified."
} }
} }
}, },

View File

@ -6641,11 +6641,18 @@ Examples:<br>
<tbody> <tbody>
<tr> <tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">path</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">path</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">The full path to the volume on the node. It can be either a directory or block device (disk, partition, &#8230;). Directories can be represented only by PersistentVolume with VolumeMode=Filesystem. Block devices can be represented only by VolumeMode=Block, which also requires the BlockVolume alpha feature gate to be enabled.</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">The full path to the volume on the node. It can be either a directory or block device (disk, partition, &#8230;).</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
<td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td>
</tr> </tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">fsType</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Filesystem type to mount. It applies only when the Path is a block device. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". The default value is to auto-select a fileystem if unspecified.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody> </tbody>
</table> </table>

View File

@ -1513,10 +1513,14 @@ type KeyToPath struct {
type LocalVolumeSource struct { type LocalVolumeSource struct {
// The full path to the volume on the node. // The full path to the volume on the node.
// It can be either a directory or block device (disk, partition, ...). // It can be either a directory or block device (disk, partition, ...).
// Directories can be represented only by PersistentVolume with VolumeMode=Filesystem.
// Block devices can be represented only by VolumeMode=Block, which also requires the
// BlockVolume alpha feature gate to be enabled.
Path string Path string
// Filesystem type to mount.
// It applies only when the Path is a block device.
// Must be a filesystem type supported by the host operating system.
// Ex. "ext4", "xfs", "ntfs". The default value is to auto-select a fileystem if unspecified.
// +optional
FSType *string
} }
// Represents storage that is managed by an external CSI volume driver (Beta feature) // Represents storage that is managed by an external CSI volume driver (Beta feature)

View File

@ -4074,6 +4074,7 @@ func Convert_core_LocalObjectReference_To_v1_LocalObjectReference(in *core.Local
func autoConvert_v1_LocalVolumeSource_To_core_LocalVolumeSource(in *v1.LocalVolumeSource, out *core.LocalVolumeSource, s conversion.Scope) error { func autoConvert_v1_LocalVolumeSource_To_core_LocalVolumeSource(in *v1.LocalVolumeSource, out *core.LocalVolumeSource, s conversion.Scope) error {
out.Path = in.Path out.Path = in.Path
out.FSType = (*string)(unsafe.Pointer(in.FSType))
return nil return nil
} }
@ -4084,6 +4085,7 @@ func Convert_v1_LocalVolumeSource_To_core_LocalVolumeSource(in *v1.LocalVolumeSo
func autoConvert_core_LocalVolumeSource_To_v1_LocalVolumeSource(in *core.LocalVolumeSource, out *v1.LocalVolumeSource, s conversion.Scope) error { func autoConvert_core_LocalVolumeSource_To_v1_LocalVolumeSource(in *core.LocalVolumeSource, out *v1.LocalVolumeSource, s conversion.Scope) error {
out.Path = in.Path out.Path = in.Path
out.FSType = (*string)(unsafe.Pointer(in.FSType))
return nil return nil
} }

View File

@ -1959,6 +1959,11 @@ func (in *LocalObjectReference) DeepCopy() *LocalObjectReference {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LocalVolumeSource) DeepCopyInto(out *LocalVolumeSource) { func (in *LocalVolumeSource) DeepCopyInto(out *LocalVolumeSource) {
*out = *in *out = *in
if in.FSType != nil {
in, out := &in.FSType, &out.FSType
*out = new(string)
**out = **in
}
return return
} }
@ -2884,7 +2889,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) {
if in.Local != nil { if in.Local != nil {
in, out := &in.Local, &out.Local in, out := &in.Local, &out.Local
*out = new(LocalVolumeSource) *out = new(LocalVolumeSource)
**out = **in (*in).DeepCopyInto(*out)
} }
if in.StorageOS != nil { if in.StorageOS != nil {
in, out := &in.StorageOS, &out.StorageOS in, out := &in.StorageOS, &out.StorageOS

View File

@ -194,7 +194,7 @@ func (f *FakeMounter) GetFileType(pathname string) (FileType, error) {
if t, ok := f.Filesystem[pathname]; ok { if t, ok := f.Filesystem[pathname]; ok {
return t, nil return t, nil
} }
return FileType("fake"), nil return FileType("Directory"), nil
} }
func (f *FakeMounter) MakeDir(pathname string) error { func (f *FakeMounter) MakeDir(pathname string) error {

View File

@ -331,8 +331,8 @@ func HasMountRefs(mountPath string, mountRefs []string) bool {
return count > 0 return count > 0
} }
// pathWithinBase checks if give path is within given base directory. // PathWithinBase checks if give path is within given base directory.
func pathWithinBase(fullPath, basePath string) bool { func PathWithinBase(fullPath, basePath string) bool {
rel, err := filepath.Rel(basePath, fullPath) rel, err := filepath.Rel(basePath, fullPath)
if err != nil { if err != nil {
return false return false

View File

@ -665,7 +665,7 @@ func findMountInfo(path, mountInfoPath string) (mountInfo, error) {
// point that is prefix of 'path' - that's the mount where path resides // point that is prefix of 'path' - that's the mount where path resides
var info *mountInfo var info *mountInfo
for i := len(infos) - 1; i >= 0; i-- { for i := len(infos) - 1; i >= 0; i-- {
if pathWithinBase(path, infos[i].mountPoint) { if PathWithinBase(path, infos[i].mountPoint) {
info = &infos[i] info = &infos[i]
break break
} }
@ -736,7 +736,7 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string,
// This implementation is shared between Linux and NsEnterMounter // This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !pathWithinBase(subpath.Path, subpath.VolumePath) { if !PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
} }
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
@ -964,7 +964,7 @@ func cleanSubPath(mounter Interface, subpath Subpath) error {
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory // removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content // if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error { func removeEmptyDirs(baseDir, endDir string) error {
if !pathWithinBase(endDir, baseDir) { if !PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
} }
@ -1052,7 +1052,7 @@ func getMode(pathname string) (os.FileMode, error) {
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
glog.V(4).Infof("Creating directory %q within base %q", pathname, base) glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !pathWithinBase(pathname, base) { if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
} }
@ -1079,7 +1079,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
if err != nil { if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err) return fmt.Errorf("error opening directory %s: %s", existingPath, err)
} }
if !pathWithinBase(fullExistingPath, base) { if !PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
} }
@ -1241,7 +1241,7 @@ func doSafeOpen(pathname string, base string) (int, error) {
// sure the user cannot change already existing directories into symlinks. // sure the user cannot change already existing directories into symlinks.
for _, seg := range segments { for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg) currentPath = filepath.Join(currentPath, seg)
if !pathWithinBase(currentPath, base) { if !PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
} }
@ -1298,7 +1298,7 @@ func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) {
// We need search in backward order because it's possible for later mounts // We need search in backward order because it's possible for later mounts
// to overlap earlier mounts. // to overlap earlier mounts.
for i := len(mis) - 1; i >= 0; i-- { for i := len(mis) - 1; i >= 0; i-- {
if hostSource == mis[i].mountPoint || pathWithinBase(hostSource, mis[i].mountPoint) { if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) {
// If it's a mount point or path under a mount point. // If it's a mount point or path under a mount point.
mountID = mis[i].id mountID = mis[i].id
rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint)) rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint))

View File

@ -413,7 +413,7 @@ func TestPathWithinBase(t *testing.T) {
}, },
} }
for _, test := range tests { for _, test := range tests {
if pathWithinBase(test.fullPath, test.basePath) != test.expected { if PathWithinBase(test.fullPath, test.basePath) != test.expected {
t.Errorf("test %q failed: expected %v", test.name, test.expected) t.Errorf("test %q failed: expected %v", test.name, test.expected)
} }

View File

@ -309,7 +309,7 @@ func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, e
break break
} }
if !pathWithinBase(currentFullPath, volumePath) { if !PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break break
} }
@ -499,7 +499,7 @@ func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
glog.V(4).Infof("Creating directory %q within base %q", pathname, base) glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !pathWithinBase(pathname, base) { if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
} }
@ -534,7 +534,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
if err != nil { if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err) return fmt.Errorf("cannot read link %s: %s", base, err)
} }
if !pathWithinBase(fullExistingPath, fullBasePath) { if !PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
} }

View File

@ -576,8 +576,8 @@ func TestPathWithinBase(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
result := pathWithinBase(test.fullPath, test.basePath) result := PathWithinBase(test.fullPath, test.basePath)
assert.Equal(t, result, test.expectedResult, "Expect result not equal with pathWithinBase(%s, %s) return: %q, expected: %q", assert.Equal(t, result, test.expectedResult, "Expect result not equal with PathWithinBase(%s, %s) return: %q, expected: %q",
test.fullPath, test.basePath, result, test.expectedResult) test.fullPath, test.basePath, result, test.expectedResult)
} }
} }

View File

@ -320,7 +320,7 @@ func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.F
evaluatedBase = filepath.Clean(evaluatedBase) evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir) rootDir := filepath.Clean(mounter.rootDir)
if pathWithinBase(evaluatedBase, rootDir) { if PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the // Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'. // container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still // This is useful when /rootfs is mounted as read-only - we can still

View File

@ -33,6 +33,7 @@ go_test(
embed = [":go_default_library"], embed = [":go_default_library"],
deps = select({ deps = select({
"@io_bazel_rules_go//go/platform:darwin": [ "@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library", "//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library", "//pkg/volume/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
@ -41,6 +42,7 @@ go_test(
"//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library",
], ],
"@io_bazel_rules_go//go/platform:linux": [ "@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library", "//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library", "//pkg/volume/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
@ -49,6 +51,7 @@ go_test(
"//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library",
], ],
"@io_bazel_rules_go//go/platform:windows": [ "@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library", "//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library", "//pkg/volume/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",

View File

@ -38,6 +38,10 @@ import (
"k8s.io/kubernetes/pkg/volume/validation" "k8s.io/kubernetes/pkg/volume/validation"
) )
const (
defaultFSType = "ext4"
)
// This is the primary entrypoint for volume plugins. // This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin { func ProbeVolumePlugins() []volume.VolumePlugin {
return []volume.VolumePlugin{&localVolumePlugin{}} return []volume.VolumePlugin{&localVolumePlugin{}}
@ -111,6 +115,11 @@ func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ vo
return nil, err return nil, err
} }
globalLocalPath, err := plugin.getGlobalLocalPath(spec)
if err != nil {
return nil, err
}
return &localVolumeMounter{ return &localVolumeMounter{
localVolume: &localVolume{ localVolume: &localVolume{
pod: pod, pod: pod,
@ -118,7 +127,7 @@ func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ vo
volName: spec.Name(), volName: spec.Name(),
mounter: plugin.host.GetMounter(plugin.GetPluginName()), mounter: plugin.host.GetMounter(plugin.GetPluginName()),
plugin: plugin, plugin: plugin,
globalPath: volumeSource.Path, globalPath: globalLocalPath,
MetricsProvider: volume.NewMetricsStatFS(volumeSource.Path), MetricsProvider: volume.NewMetricsStatFS(volumeSource.Path),
}, },
readOnly: readOnly, readOnly: readOnly,
@ -207,6 +216,163 @@ func (plugin *localVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volu
return volume.NewSpecFromPersistentVolume(localVolume, false), nil return volume.NewSpecFromPersistentVolume(localVolume, false), nil
} }
func (plugin *localVolumePlugin) generateBlockDeviceBaseGlobalPath() string {
return filepath.Join(plugin.host.GetPluginDir(localVolumePluginName), mount.MountsInGlobalPDPath)
}
func (plugin *localVolumePlugin) getGlobalLocalPath(spec *volume.Spec) (string, error) {
if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
return "", fmt.Errorf("local volume source is nil or local path is not set")
}
fileType, err := plugin.host.GetMounter(plugin.GetPluginName()).GetFileType(spec.PersistentVolume.Spec.Local.Path)
if err != nil {
return "", err
}
switch fileType {
case mount.FileTypeDirectory:
return spec.PersistentVolume.Spec.Local.Path, nil
case mount.FileTypeBlockDev:
return filepath.Join(plugin.generateBlockDeviceBaseGlobalPath(), spec.Name()), nil
default:
return "", fmt.Errorf("only directory and block device are supported")
}
}
var _ volume.DeviceMountableVolumePlugin = &localVolumePlugin{}
type deviceMounter struct {
plugin *localVolumePlugin
mounter *mount.SafeFormatAndMount
}
var _ volume.DeviceMounter = &deviceMounter{}
func (plugin *localVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
return &deviceMounter{
plugin: plugin,
mounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
}, nil
}
func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
glog.V(4).Infof("local: mounting device %s to %s", devicePath, deviceMountPath)
notMnt, err := dm.mounter.IsLikelyNotMountPoint(deviceMountPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
return err
}
notMnt = true
} else {
return err
}
}
if !notMnt {
return nil
}
fstype, err := getVolumeSourceFSType(spec)
if err != nil {
return err
}
ro, err := getVolumeSourceReadOnly(spec)
if err != nil {
return err
}
options := []string{}
if ro {
options = append(options, "ro")
}
mountOptions := util.MountOptionFromSpec(spec, options...)
err = dm.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
if err != nil {
os.Remove(deviceMountPath)
return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %v", devicePath, deviceMountPath, fstype, err)
}
glog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype)
return nil
}
func (dm *deviceMounter) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
return fmt.Errorf("local volume source is nil or local path is not set")
}
fileType, err := dm.mounter.GetFileType(spec.PersistentVolume.Spec.Local.Path)
if err != nil {
return err
}
switch fileType {
case mount.FileTypeBlockDev:
// local volume plugin does not implement AttachableVolumePlugin interface, so set devicePath to Path in PV spec directly
devicePath = spec.PersistentVolume.Spec.Local.Path
return dm.mountLocalBlockDevice(spec, devicePath, deviceMountPath)
case mount.FileTypeDirectory:
// if the given local volume path is of already filesystem directory, return directly
return nil
default:
return fmt.Errorf("only directory and block device are supported")
}
}
func getVolumeSourceFSType(spec *volume.Spec) (string, error) {
if spec.PersistentVolume != nil &&
spec.PersistentVolume.Spec.Local != nil {
if spec.PersistentVolume.Spec.Local.FSType != nil {
return *spec.PersistentVolume.Spec.Local.FSType, nil
} else {
// if the FSType is not set in local PV spec, setting it to default ("ext4")
return defaultFSType, nil
}
}
return "", fmt.Errorf("spec does not reference a Local volume type")
}
func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) {
if spec.PersistentVolume != nil &&
spec.PersistentVolume.Spec.Local != nil {
// local volumes used as a PersistentVolume gets the ReadOnly flag indirectly through
// the persistent-claim volume used to mount the PV
return spec.ReadOnly, nil
}
return false, fmt.Errorf("spec does not reference a Local volume type")
}
func (dm *deviceMounter) GetDeviceMountPath(spec *volume.Spec) (string, error) {
return dm.plugin.getGlobalLocalPath(spec)
}
func (plugin *localVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
return &deviceMounter{
plugin: plugin,
mounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
}, nil
}
func (plugin *localVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
mounter := plugin.host.GetMounter(plugin.GetPluginName())
return mounter.GetMountRefs(deviceMountPath)
}
var _ volume.DeviceUnmounter = &deviceMounter{}
func (dm *deviceMounter) UnmountDevice(deviceMountPath string) error {
// If the local PV is a block device,
// The deviceMountPath is generated to the format like :/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/localpv.spec.Name;
// If it is a filesystem directory, then the deviceMountPath is set directly to pvSpec.Local.Path
// We only need to unmount block device here, so we need to check if the deviceMountPath passed here
// has base mount path: /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts
basemountPath := dm.plugin.generateBlockDeviceBaseGlobalPath()
if mount.PathWithinBase(deviceMountPath, basemountPath) {
return util.UnmountPath(deviceMountPath, dm.mounter)
}
return nil
}
// Local volumes represent a local directory on a node. // Local volumes represent a local directory on a node.
// The directory at the globalPath will be bind-mounted to the pod's directory // The directory at the globalPath will be bind-mounted to the pod's directory
type localVolume struct { type localVolume struct {

View File

@ -31,16 +31,18 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
utiltesting "k8s.io/client-go/util/testing" utiltesting "k8s.io/client-go/util/testing"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume"
volumetest "k8s.io/kubernetes/pkg/volume/testing" volumetest "k8s.io/kubernetes/pkg/volume/testing"
) )
const ( const (
testPVName = "pvA" testPVName = "pvA"
testMountPath = "pods/poduid/volumes/kubernetes.io~local-volume/pvA" testMountPath = "pods/poduid/volumes/kubernetes.io~local-volume/pvA"
testGlobalPath = "plugins/kubernetes.io~local-volume/volumeDevices/pvA" testGlobalPath = "plugins/kubernetes.io~local-volume/volumeDevices/pvA"
testPodPath = "pods/poduid/volumeDevices/kubernetes.io~local-volume" testPodPath = "pods/poduid/volumeDevices/kubernetes.io~local-volume"
testNodeName = "fakeNodeName" testNodeName = "fakeNodeName"
testBlockFormattingToFSGlobalPath = "plugins/kubernetes.io/local-volume/mounts/pvA"
) )
func getPlugin(t *testing.T) (string, volume.VolumePlugin) { func getPlugin(t *testing.T) (string, volume.VolumePlugin) {
@ -102,6 +104,33 @@ func getPersistentPlugin(t *testing.T) (string, volume.PersistentVolumePlugin) {
return tmpDir, plug return tmpDir, plug
} }
func getDeviceMountablePluginWithBlockPath(t *testing.T, isBlockDevice bool) (string, volume.DeviceMountableVolumePlugin) {
tmpDir, err := utiltesting.MkTmpdir("localVolumeTest")
if err != nil {
t.Fatalf("can't make a temp dir: %v", err)
}
plugMgr := volume.VolumePluginMgr{}
var pathToFSType map[string]mount.FileType
if isBlockDevice {
pathToFSType = map[string]mount.FileType{
tmpDir: mount.FileTypeBlockDev,
}
}
plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHostWithMounterFSType(tmpDir, nil, nil, pathToFSType))
plug, err := plugMgr.FindDeviceMountablePluginByName(localVolumePluginName)
if err != nil {
os.RemoveAll(tmpDir)
t.Fatalf("Can't find the plugin by name")
}
if plug.GetPluginName() != localVolumePluginName {
t.Errorf("Wrong name: %s", plug.GetPluginName())
}
return tmpDir, plug
}
func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec { func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec {
pv := &v1.PersistentVolume{ pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -179,6 +208,87 @@ func TestInvalidLocalPath(t *testing.T) {
} }
} }
func TestBlockDeviceGlobalPathAndMountDevice(t *testing.T) {
// Block device global mount path testing
tmpBlockDir, plug := getDeviceMountablePluginWithBlockPath(t, true)
defer os.RemoveAll(tmpBlockDir)
dm, err := plug.NewDeviceMounter()
if err != nil {
t.Errorf("Failed to make a new device mounter: %v", err)
}
pvSpec := getTestVolume(false, tmpBlockDir, false)
expectedGlobalPath := filepath.Join(tmpBlockDir, testBlockFormattingToFSGlobalPath)
actualPath, err := dm.GetDeviceMountPath(pvSpec)
if err != nil {
t.Errorf("Failed to get device mount path: %v", err)
}
if expectedGlobalPath != actualPath {
t.Fatalf("Expected device mount global path:%s, got: %s", expectedGlobalPath, actualPath)
}
fmt.Println("expected global path is:", expectedGlobalPath)
err = dm.MountDevice(pvSpec, tmpBlockDir, expectedGlobalPath)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(actualPath); err != nil {
if os.IsNotExist(err) {
t.Errorf("DeviceMounter.MountDevice() failed, device mount path not created: %s", actualPath)
} else {
t.Errorf("DeviceMounter.MountDevice() failed: %v", err)
}
}
du, err := plug.NewDeviceUnmounter()
if err != nil {
t.Fatalf("Create device unmounter error: %v", err)
}
err = du.UnmountDevice(actualPath)
if err != nil {
t.Fatalf("Unmount device error: %v", err)
}
}
func TestFSGlobalPathAndMountDevice(t *testing.T) {
// FS global path testing
tmpFSDir, plug := getDeviceMountablePluginWithBlockPath(t, false)
defer os.RemoveAll(tmpFSDir)
dm, err := plug.NewDeviceMounter()
if err != nil {
t.Errorf("Failed to make a new device mounter: %v", err)
}
pvSpec := getTestVolume(false, tmpFSDir, false)
expectedGlobalPath := tmpFSDir
actualPath, err := dm.GetDeviceMountPath(pvSpec)
if err != nil {
t.Errorf("Failed to get device mount path: %v", err)
}
if expectedGlobalPath != actualPath {
t.Fatalf("Expected device mount global path:%s, got: %s", expectedGlobalPath, actualPath)
}
// Actually, we will do nothing if the local path is FS type
err = dm.MountDevice(pvSpec, tmpFSDir, expectedGlobalPath)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(expectedGlobalPath); err != nil {
if os.IsNotExist(err) {
t.Errorf("DeviceMounter.MountDevice() failed, device mount path not created: %s", expectedGlobalPath)
} else {
t.Errorf("DeviceMounter.MountDevice() failed: %v", err)
}
}
}
func TestMountUnmount(t *testing.T) { func TestMountUnmount(t *testing.T) {
tmpDir, plug := getPlugin(t) tmpDir, plug := getPlugin(t)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)

View File

@ -60,33 +60,40 @@ type fakeVolumeHost struct {
} }
func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost { func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost {
return newFakeVolumeHost(rootDir, kubeClient, plugins, nil) return newFakeVolumeHost(rootDir, kubeClient, plugins, nil, nil)
} }
func NewFakeVolumeHostWithCloudProvider(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface) *fakeVolumeHost { func NewFakeVolumeHostWithCloudProvider(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface) *fakeVolumeHost {
return newFakeVolumeHost(rootDir, kubeClient, plugins, cloud) return newFakeVolumeHost(rootDir, kubeClient, plugins, cloud, nil)
} }
func NewFakeVolumeHostWithNodeLabels(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, labels map[string]string) *fakeVolumeHost { func NewFakeVolumeHostWithNodeLabels(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, labels map[string]string) *fakeVolumeHost {
volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil) volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil, nil)
volHost.nodeLabels = labels volHost.nodeLabels = labels
return volHost return volHost
} }
func NewFakeVolumeHostWithNodeName(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, nodeName string) *fakeVolumeHost { func NewFakeVolumeHostWithNodeName(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, nodeName string) *fakeVolumeHost {
volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil) volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil, nil)
volHost.nodeName = nodeName volHost.nodeName = nodeName
return volHost return volHost
} }
func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface) *fakeVolumeHost { func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface, pathToTypeMap map[string]mount.FileType) *fakeVolumeHost {
host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: cloud} host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: cloud}
host.mounter = &mount.FakeMounter{} host.mounter = &mount.FakeMounter{
Filesystem: pathToTypeMap,
}
host.exec = mount.NewFakeExec(nil) host.exec = mount.NewFakeExec(nil)
host.pluginMgr.InitPlugins(plugins, nil /* prober */, host) host.pluginMgr.InitPlugins(plugins, nil /* prober */, host)
return host return host
} }
func NewFakeVolumeHostWithMounterFSType(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, pathToTypeMap map[string]mount.FileType) *fakeVolumeHost {
volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil, pathToTypeMap)
return volHost
}
func (f *fakeVolumeHost) GetPluginDir(podUID string) string { func (f *fakeVolumeHost) GetPluginDir(podUID string) string {
return path.Join(f.rootDir, "plugins", podUID) return path.Join(f.rootDir, "plugins", podUID)
} }

File diff suppressed because it is too large Load Diff

View File

@ -1724,10 +1724,14 @@ message LocalObjectReference {
message LocalVolumeSource { message LocalVolumeSource {
// The full path to the volume on the node. // The full path to the volume on the node.
// It can be either a directory or block device (disk, partition, ...). // It can be either a directory or block device (disk, partition, ...).
// Directories can be represented only by PersistentVolume with VolumeMode=Filesystem.
// Block devices can be represented only by VolumeMode=Block, which also requires the
// BlockVolume alpha feature gate to be enabled.
optional string path = 1; optional string path = 1;
// Filesystem type to mount.
// It applies only when the Path is a block device.
// Must be a filesystem type supported by the host operating system.
// Ex. "ext4", "xfs", "ntfs". The default value is to auto-select a fileystem if unspecified.
// +optional
optional string fsType = 2;
} }
// Represents an NFS mount that lasts the lifetime of a pod. // Represents an NFS mount that lasts the lifetime of a pod.

View File

@ -1601,10 +1601,14 @@ type KeyToPath struct {
type LocalVolumeSource struct { type LocalVolumeSource struct {
// The full path to the volume on the node. // The full path to the volume on the node.
// It can be either a directory or block device (disk, partition, ...). // It can be either a directory or block device (disk, partition, ...).
// Directories can be represented only by PersistentVolume with VolumeMode=Filesystem.
// Block devices can be represented only by VolumeMode=Block, which also requires the
// BlockVolume alpha feature gate to be enabled.
Path string `json:"path" protobuf:"bytes,1,opt,name=path"` Path string `json:"path" protobuf:"bytes,1,opt,name=path"`
// Filesystem type to mount.
// It applies only when the Path is a block device.
// Must be a filesystem type supported by the host operating system.
// Ex. "ext4", "xfs", "ntfs". The default value is to auto-select a fileystem if unspecified.
// +optional
FSType *string `json:"fsType,omitempty" protobuf:"bytes,2,opt,name=fsType"`
} }
// Represents storage that is managed by an external CSI volume driver (Beta feature) // Represents storage that is managed by an external CSI volume driver (Beta feature)

View File

@ -891,8 +891,9 @@ func (LocalObjectReference) SwaggerDoc() map[string]string {
} }
var map_LocalVolumeSource = map[string]string{ var map_LocalVolumeSource = map[string]string{
"": "Local represents directly-attached storage with node affinity (Beta feature)", "": "Local represents directly-attached storage with node affinity (Beta feature)",
"path": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...). Directories can be represented only by PersistentVolume with VolumeMode=Filesystem. Block devices can be represented only by VolumeMode=Block, which also requires the BlockVolume alpha feature gate to be enabled.", "path": "The full path to the volume on the node. It can be either a directory or block device (disk, partition, ...).",
"fsType": "Filesystem type to mount. It applies only when the Path is a block device. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default value is to auto-select a fileystem if unspecified.",
} }
func (LocalVolumeSource) SwaggerDoc() map[string]string { func (LocalVolumeSource) SwaggerDoc() map[string]string {

View File

@ -1957,6 +1957,11 @@ func (in *LocalObjectReference) DeepCopy() *LocalObjectReference {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LocalVolumeSource) DeepCopyInto(out *LocalVolumeSource) { func (in *LocalVolumeSource) DeepCopyInto(out *LocalVolumeSource) {
*out = *in *out = *in
if in.FSType != nil {
in, out := &in.FSType, &out.FSType
*out = new(string)
**out = **in
}
return return
} }
@ -2882,7 +2887,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) {
if in.Local != nil { if in.Local != nil {
in, out := &in.Local, &out.Local in, out := &in.Local, &out.Local
*out = new(LocalVolumeSource) *out = new(LocalVolumeSource)
**out = **in (*in).DeepCopyInto(*out)
} }
if in.StorageOS != nil { if in.StorageOS != nil {
in, out := &in.StorageOS, &out.StorageOS in, out := &in.StorageOS, &out.StorageOS

View File

@ -78,8 +78,12 @@ const (
GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs" GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs"
// Creates a local file, formats it, and maps it as a block device. // Creates a local file, formats it, and maps it as a block device.
BlockLocalVolumeType localVolumeType = "block" BlockLocalVolumeType localVolumeType = "block"
// Creates a local file, formats it, and mounts it to use as local volume. // Creates a local file serving as the backing for block device., formats it,
BlockFsLocalVolumeType localVolumeType = "blockfs" // and mounts it to use as FS mode local volume.
BlockFsWithFormatLocalVolumeType localVolumeType = "blockfswithformat"
// Creates a local file serving as the backing for block device. do not format it manually,
// and mounts it to use as FS mode local volume.
BlockFsWithoutFormatLocalVolumeType localVolumeType = "blockfswithoutformat"
) )
var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{ var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{
@ -90,7 +94,8 @@ var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *
DirectoryBindMountedLocalVolumeType: setupLocalVolumeDirectoryBindMounted, DirectoryBindMountedLocalVolumeType: setupLocalVolumeDirectoryBindMounted,
DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted, DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted,
BlockLocalVolumeType: setupLocalVolumeBlock, BlockLocalVolumeType: setupLocalVolumeBlock,
BlockFsLocalVolumeType: setupLocalVolumeBlockFs, BlockFsWithFormatLocalVolumeType: setupLocalVolumeBlockFsWithFormat,
BlockFsWithoutFormatLocalVolumeType: setupLocalVolumeBlockFsWithoutFormat,
} }
var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){ var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){
@ -101,7 +106,8 @@ var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTes
DirectoryBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryBindMounted, DirectoryBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryBindMounted,
DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted, DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted,
BlockLocalVolumeType: cleanupLocalVolumeBlock, BlockLocalVolumeType: cleanupLocalVolumeBlock,
BlockFsLocalVolumeType: cleanupLocalVolumeBlockFs, BlockFsWithFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithFormat,
BlockFsWithoutFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithoutFormat,
} }
type localTestVolume struct { type localTestVolume struct {
@ -247,6 +253,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
pod1, pod1Err = createLocalPod(config, testVol, nil) pod1, pod1Err = createLocalPod(config, testVol, nil)
Expect(pod1Err).NotTo(HaveOccurred()) Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name) verifyLocalPod(config, testVol, pod1, config.node0.Name)
writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
}) })
AfterEach(func() { AfterEach(func() {
@ -256,16 +267,16 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
It("should be able to mount volume and read from pod1", func() { It("should be able to mount volume and read from pod1", func() {
By("Reading in pod1") By("Reading in pod1")
// testFileContent was written during setupLocalVolume // testFileContent was written in BeforeEach
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType) testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
}) })
It("should be able to mount volume and write from pod1", func() { It("should be able to mount volume and write from pod1", func() {
// testFileContent was written during setupLocalVolume // testFileContent was written in BeforeEach
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType) testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
By("Writing in pod1") By("Writing in pod1")
writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType) writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType)
podRWCmdExec(pod1, writeCmd) podRWCmdExec(pod1, writeCmd)
}) })
}) })
@ -346,12 +357,12 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
Context("Local volume that cannot be mounted [Slow]", func() { Context("Local volume that cannot be mounted [Slow]", func() {
// TODO: // TODO:
// - check for these errors in unit tests intead // - check for these errors in unit tests instead
It("should fail due to non-existent path", func() { It("should fail due to non-existent path", func() {
ep := &eventPatterns{ ep := &eventPatterns{
reason: "FailedMount", reason: "FailedMount",
pattern: make([]string, 2)} pattern: make([]string, 2)}
ep.pattern = append(ep.pattern, "MountVolume.SetUp failed") ep.pattern = append(ep.pattern, "MountVolume.NewMounter initialization failed")
testVol := &localTestVolume{ testVol := &localTestVolume{
node: config.node0, node: config.node0,
@ -461,7 +472,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
// Delete the persistent volume claim: file will be cleaned up and volume be re-created. // Delete the persistent volume claim: file will be cleaned up and volume be re-created.
By("Deleting the persistent volume claim to clean up persistent volume and re-create one") By("Deleting the persistent volume claim to clean up persistent volume and re-create one")
writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent, DirectoryLocalVolumeType) writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType)
err = issueNodeCommand(config, writeCmd, config.node0) err = issueNodeCommand(config, writeCmd, config.node0)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{}) err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{})
@ -731,8 +742,6 @@ func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeTyp
err = framework.WaitForPodNameUnschedulableInNamespace(config.client, pod.Name, pod.Namespace) err = framework.WaitForPodNameUnschedulableInNamespace(config.client, pod.Name, pod.Namespace)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cleanupLocalVolumes(config, []*localTestVolume{testVol})
} }
type eventPatterns struct { type eventPatterns struct {
@ -766,7 +775,12 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
Expect(pod1Err).NotTo(HaveOccurred()) Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name) verifyLocalPod(config, testVol, pod1, config.node0.Name)
// testFileContent was written during setupLocalVolume writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
// testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType) testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
By("Creating pod2 to read from the PV") By("Creating pod2 to read from the PV")
@ -774,16 +788,16 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
Expect(pod2Err).NotTo(HaveOccurred()) Expect(pod2Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod2, config.node0.Name) verifyLocalPod(config, testVol, pod2, config.node0.Name)
// testFileContent was written during setupLocalVolume // testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType) testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType) writeCmd = createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
By("Writing in pod1") By("Writing in pod2")
podRWCmdExec(pod1, writeCmd) podRWCmdExec(pod2, writeCmd)
By("Reading in pod2") By("Reading in pod1")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType) testReadFileContent(volumeDir, testFile, testVol.hostDir, pod1, testVol.localVolumeType)
By("Deleting pod1") By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name) framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@ -798,14 +812,14 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
Expect(pod1Err).NotTo(HaveOccurred()) Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name) verifyLocalPod(config, testVol, pod1, config.node0.Name)
// testFileContent was written during setupLocalVolume writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
By("Writing in pod1") By("Writing in pod1")
podRWCmdExec(pod1, writeCmd) podRWCmdExec(pod1, writeCmd)
// testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
By("Deleting pod1") By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name) framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@ -815,7 +829,7 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
verifyLocalPod(config, testVol, pod2, config.node0.Name) verifyLocalPod(config, testVol, pod2, config.node0.Name)
By("Reading in pod2") By("Reading in pod2")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType) testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
By("Deleting pod2") By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name) framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
@ -885,11 +899,13 @@ func cleanupLocalVolumes(config *localTestConfig, volumes []*localTestVolume) {
} }
} }
func setupWriteTestFile(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume { func generateLocalTestVolume(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume {
writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent, localVolumeType) if localVolumeType != BlockLocalVolumeType && localVolumeType != BlockFsWithoutFormatLocalVolumeType {
By(fmt.Sprintf("Creating test file on node %q in path %q", node.Name, hostDir)) mkdirCmd := fmt.Sprintf("mkdir -p %s", hostDir)
err := issueNodeCommand(config, writeCmd, node) err := issueNodeCommand(config, mkdirCmd, node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}
return &localTestVolume{ return &localTestVolume{
node: node, node: node,
hostDir: hostDir, hostDir: hostDir,
@ -901,8 +917,7 @@ func setupLocalVolumeTmpfs(config *localTestConfig, node *v1.Node) *localTestVol
testDirName := "local-volume-test-" + string(uuid.NewUUID()) testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
createAndMountTmpfsLocalVolume(config, hostDir, node) createAndMountTmpfsLocalVolume(config, hostDir, node)
// populate volume with testFile containing testFileContent return generateLocalTestVolume(hostDir, config, TmpfsLocalVolumeType, node)
return setupWriteTestFile(hostDir, config, TmpfsLocalVolumeType, node)
} }
func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -910,15 +925,13 @@ func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localT
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
dirName := strings.Fields(res)[0] dirName := strings.Fields(res)[0]
hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName
// Populate volume with testFile containing testFileContent. return generateLocalTestVolume(hostDir, config, GCELocalSSDVolumeType, node)
return setupWriteTestFile(hostDir, config, GCELocalSSDVolumeType, node)
} }
func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID()) testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
// Populate volume with testFile containing testFileContent. return generateLocalTestVolume(hostDir, config, DirectoryLocalVolumeType, node)
return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node)
} }
// launchNodeExecPodForLocalPV launches a hostexec pod for local PV and waits // launchNodeExecPodForLocalPV launches a hostexec pod for local PV and waits
@ -992,11 +1005,10 @@ func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *loca
testDirName := "local-volume-test-" + string(uuid.NewUUID()) testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
hostDirBackend := hostDir + "-backend" hostDirBackend := hostDir + "-backend"
cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir) cmd := fmt.Sprintf("mkdir %s && sudo ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node) _, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent. return generateLocalTestVolume(hostDir, config, DirectoryLinkLocalVolumeType, node)
return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node)
} }
func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -1005,20 +1017,18 @@ func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node
cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir) cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node) _, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent. return generateLocalTestVolume(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
} }
func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID()) testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
hostDirBackend := hostDir + "-backend" hostDirBackend := hostDir + "-backend"
cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s", cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && sudo ln -s %s %s",
hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir) hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node) _, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent. return generateLocalTestVolume(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
} }
func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -1026,14 +1036,13 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node) createAndMapBlockLocalVolume(config, hostDir, node)
loopDev := getBlockLoopDev(config, hostDir, node) loopDev := getBlockLoopDev(config, hostDir, node)
// Populate block volume with testFile containing testFileContent. volume := generateLocalTestVolume(loopDev, config, BlockLocalVolumeType, node)
volume := setupWriteTestFile(loopDev, config, BlockLocalVolumeType, node)
volume.hostDir = loopDev volume.hostDir = loopDev
volume.loopDevDir = hostDir volume.loopDevDir = hostDir
return volume return volume
} }
func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume { func setupLocalVolumeBlockFsWithFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID()) testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName) hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node) createAndMapBlockLocalVolume(config, hostDir, node)
@ -1043,13 +1052,25 @@ func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestV
cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir) cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node) _, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Populate block volume with testFile containing testFileContent. volume := generateLocalTestVolume(hostDir, config, BlockFsWithFormatLocalVolumeType, node)
volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node)
volume.hostDir = hostDir volume.hostDir = hostDir
volume.loopDevDir = loopDev volume.loopDevDir = loopDev
return volume return volume
} }
func setupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node)
loopDev := getBlockLoopDev(config, hostDir, node)
volume := generateLocalTestVolume(loopDev, config, BlockFsWithoutFormatLocalVolumeType, node)
// we do this in order to set block device path to local PV spec path directly
// and test local volume plugin FileSystem mode on block device
volume.hostDir = loopDev
volume.loopDevDir = hostDir
return volume
}
// Determine the /dev/loopXXX device associated with this test, via its hostDir. // Determine the /dev/loopXXX device associated with this test, via its hostDir.
func getBlockLoopDev(config *localTestConfig, hostDir string, node *v1.Node) string { func getBlockLoopDev(config *localTestConfig, hostDir string, node *v1.Node) string {
loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir) loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir)
@ -1100,7 +1121,7 @@ func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestV
By("Removing the test directory") By("Removing the test directory")
hostDir := volume.hostDir hostDir := volume.hostDir
hostDirBackend := hostDir + "-backend" hostDirBackend := hostDir + "-backend"
removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend) removeCmd := fmt.Sprintf("sudo rm -r %s && rm -r %s", hostDir, hostDirBackend)
err := issueNodeCommand(config, removeCmd, volume.node) err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
@ -1119,7 +1140,7 @@ func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume
By("Removing the test directory") By("Removing the test directory")
hostDir := volume.hostDir hostDir := volume.hostDir
hostDirBackend := hostDir + "-backend" hostDirBackend := hostDir + "-backend"
removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend) removeCmd := fmt.Sprintf("sudo rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
err := issueNodeCommand(config, removeCmd, volume.node) err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
@ -1135,7 +1156,7 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) {
} }
// Deletes the PVC/PV and removes the test directory holding the block file. // Deletes the PVC/PV and removes the test directory holding the block file.
func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) { func cleanupLocalVolumeBlockFsWithFormat(config *localTestConfig, volume *localTestVolume) {
// umount first // umount first
By("Umount blockfs mountpoint") By("Umount blockfs mountpoint")
umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir) umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir)
@ -1147,6 +1168,15 @@ func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
func cleanupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, volume *localTestVolume) {
volume.hostDir = volume.loopDevDir
unmapBlockLocalVolume(config, volume.hostDir, volume.node)
By("Removing the test directory")
removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred())
}
func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig { func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig {
pvcConfig := framework.PersistentVolumeClaimConfig{ pvcConfig := framework.PersistentVolumeClaimConfig{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
@ -1337,14 +1367,6 @@ func unmapBlockLocalVolume(config *localTestConfig, dir string, node *v1.Node) {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
// Create corresponding write and read commands
// to be executed via hostexec Pod on the node with the local PV
func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) (writeCmd string, readCmd string) {
writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent, volumeType)
readCmd = createReadCmd(testFileDir, testFile, volumeType)
return writeCmd, readCmd
}
func createWriteCmd(testDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) string { func createWriteCmd(testDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) string {
if volumeType == BlockLocalVolumeType { if volumeType == BlockLocalVolumeType {
// testDir is the block device. // testDir is the block device.
@ -1378,7 +1400,7 @@ func createReadCmd(testFileDir string, testFile string, volumeType localVolumeTy
// Read testFile and evaluate whether it contains the testFileContent // Read testFile and evaluate whether it contains the testFileContent
func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod, volumeType localVolumeType) { func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod, volumeType localVolumeType) {
readCmd := createReadCmd(volumeDir, testFile, volumeType) readCmd := createReadCmd(testFileDir, testFile, volumeType)
readOut := podRWCmdExec(pod, readCmd) readOut := podRWCmdExec(pod, readCmd)
Expect(readOut).To(ContainSubstring(testFileContent)) Expect(readOut).To(ContainSubstring(testFileContent))
} }