diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index ca6d0a5d3e0..c080e7b5fdf 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -45,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) @@ -136,6 +137,7 @@ func NewAttachDetachController( eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "attachdetach-controller"}) + blkutil := volumeutil.NewBlockVolumePathHandler() adc.desiredStateOfWorld = cache.NewDesiredStateOfWorld(&adc.volumePluginMgr) adc.actualStateOfWorld = cache.NewActualStateOfWorld(&adc.volumePluginMgr) @@ -144,7 +146,8 @@ func NewAttachDetachController( kubeClient, &adc.volumePluginMgr, recorder, - false)) // flag for experimental binary check for volume mount + false, // flag for experimental binary check for volume mount + blkutil)) adc.nodeStatusUpdater = statusupdater.NewNodeStatusUpdater( kubeClient, nodeInformer.Lister(), adc.actualStateOfWorld) @@ -515,6 +518,10 @@ func (adc *attachDetachController) GetPluginDir(podUID string) string { return "" } +func (adc *attachDetachController) GetVolumeDevicePluginDir(podUID string) string { + return "" +} + func (adc *attachDetachController) GetPodVolumeDir(podUID types.UID, pluginName, volumeName string) string { return "" } @@ -523,6 +530,10 @@ func (adc *attachDetachController) GetPodPluginDir(podUID types.UID, pluginName return "" } +func (adc *attachDetachController) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string { + return "" +} + func (adc *attachDetachController) GetKubeClient() clientset.Interface { return adc.kubeClient } diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go index 79bbc520fe1..9d7f2aefbdf 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go @@ -50,7 +50,13 @@ func Test_Run_Positive_DoNothing(t *testing.T) { asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc()) nsu := statusupdater.NewNodeStatusUpdater( fakeKubeClient, informerFactory.Core().V1().Nodes().Lister(), asw) @@ -80,7 +86,13 @@ func Test_Run_Positive_OneDesiredVolumeAttach(t *testing.T) { asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) @@ -126,7 +138,13 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithUnmountedVolume(t *te asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) @@ -193,7 +211,13 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithMountedVolume(t *test asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) @@ -260,7 +284,13 @@ func Test_Run_Negative_OneDesiredVolumeAttachThenDetachWithUnmountedVolumeUpdate asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(true /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) @@ -330,7 +360,13 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing. asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) @@ -416,7 +452,13 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing. asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} - ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + fakeKubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) diff --git a/pkg/controller/volume/expand/expand_controller.go b/pkg/controller/volume/expand/expand_controller.go index 1c71528ef91..de16d73206e 100644 --- a/pkg/controller/volume/expand/expand_controller.go +++ b/pkg/controller/volume/expand/expand_controller.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" ) @@ -117,12 +118,14 @@ func NewExpandController( eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "volume_expand"}) + blkutil := util.NewBlockVolumePathHandler() expc.opExecutor = operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( kubeClient, &expc.volumePluginMgr, recorder, - false)) + false, + blkutil)) expc.resizeMap = cache.NewVolumeResizeMap(expc.kubeClient) @@ -203,10 +206,18 @@ func (expc *expandController) GetPluginDir(pluginName string) string { return "" } +func (expc *expandController) GetVolumeDevicePluginDir(pluginName string) string { + return "" +} + func (expc *expandController) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string { return "" } +func (expc *expandController) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string { + return "" +} + func (expc *expandController) GetPodPluginDir(podUID types.UID, pluginName string) string { return "" } diff --git a/pkg/controller/volume/persistentvolume/volume_host.go b/pkg/controller/volume/persistentvolume/volume_host.go index 8f182edc6b9..d111ed07111 100644 --- a/pkg/controller/volume/persistentvolume/volume_host.go +++ b/pkg/controller/volume/persistentvolume/volume_host.go @@ -37,6 +37,10 @@ func (ctrl *PersistentVolumeController) GetPluginDir(pluginName string) string { return "" } +func (ctrl *PersistentVolumeController) GetVolumeDevicePluginDir(pluginName string) string { + return "" +} + func (ctrl *PersistentVolumeController) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string { return "" } @@ -45,6 +49,10 @@ func (ctrl *PersistentVolumeController) GetPodPluginDir(podUID types.UID, plugin return "" } +func (ctrl *PersistentVolumeController) GetPodVolumeDeviceDir(ppodUID types.UID, pluginName string) string { + return "" +} + func (ctrl *PersistentVolumeController) GetKubeClient() clientset.Interface { return ctrl.kubeClient } diff --git a/pkg/kubelet/config/defaults.go b/pkg/kubelet/config/defaults.go index e70659d3675..b90306968f1 100644 --- a/pkg/kubelet/config/defaults.go +++ b/pkg/kubelet/config/defaults.go @@ -19,6 +19,7 @@ package config const ( DefaultKubeletPodsDirName = "pods" DefaultKubeletVolumesDirName = "volumes" + DefaultKubeletVolumeDevicesDirName = "volumeDevices" DefaultKubeletPluginsDirName = "plugins" DefaultKubeletContainersDirName = "containers" DefaultKubeletPluginContainersDirName = "plugin-containers" diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 4e372d639a7..2f5f4e3d188 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -446,9 +446,14 @@ type RunContainerOptions struct { type VolumeInfo struct { // Mounter is the volume's mounter Mounter volume.Mounter + // BlockVolumeMapper is the Block volume's mapper + BlockVolumeMapper volume.BlockVolumeMapper // SELinuxLabeled indicates whether this volume has had the // pod's SELinux label applied to it or not SELinuxLabeled bool + // Whether the volume permission is set to read-only or not + // This value is passed from volume.spec + ReadOnly bool } type VolumeMap map[string]VolumeInfo diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index 5ef68d9b448..59c5e9c5ba1 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -53,6 +53,8 @@ const ( FailedMountVolume = "FailedMount" VolumeResizeFailed = "VolumeResizeFailed" FailedUnMountVolume = "FailedUnMount" + FailedMapVolume = "FailedMapVolume" + FailedUnmapDevice = "FailedUnmapDevice" WarnAlreadyMountedVolume = "AlreadyMountedVolume" SuccessfulDetachVolume = "SuccessfulDetachVolume" SuccessfulMountVolume = "SuccessfulMountVolume" diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index 6519d752272..fefd8aac144 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -64,6 +64,21 @@ func (kl *Kubelet) getPluginDir(pluginName string) string { return filepath.Join(kl.getPluginsDir(), pluginName) } +// getVolumeDevicePluginsDir returns the full path to the directory under which plugin +// directories are created. Plugins can use these directories for data that +// they need to persist. Plugins should create subdirectories under this named +// after their own names. +func (kl *Kubelet) getVolumeDevicePluginsDir() string { + return filepath.Join(kl.getRootDir(), config.DefaultKubeletPluginsDirName) +} + +// getVolumeDevicePluginDir returns a data directory name for a given plugin name. +// Plugins can use these directories to store data that they need to persist. +// For per-pod plugin data, see getVolumeDevicePluginsDir. +func (kl *Kubelet) getVolumeDevicePluginDir(pluginName string) string { + return filepath.Join(kl.getVolumeDevicePluginsDir(), pluginName, config.DefaultKubeletVolumeDevicesDirName) +} + // GetPodDir returns the full path to the per-pod data directory for the // specified pod. This directory may not exist if the pod does not exist. func (kl *Kubelet) GetPodDir(podUID types.UID) string { @@ -90,6 +105,19 @@ func (kl *Kubelet) getPodVolumeDir(podUID types.UID, pluginName string, volumeNa return filepath.Join(kl.getPodVolumesDir(podUID), pluginName, volumeName) } +// getPodVolumeDevicesDir returns the full path to the per-pod data directory under +// which volumes are created for the specified pod. This directory may not +// exist if the pod does not exist. +func (kl *Kubelet) getPodVolumeDevicesDir(podUID types.UID) string { + return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletVolumeDevicesDirName) +} + +// getPodVolumeDeviceDir returns the full path to the directory which represents the +// named plugin for specified pod. This directory may not exist if the pod does not exist. +func (kl *Kubelet) getPodVolumeDeviceDir(podUID types.UID, pluginName string) string { + return filepath.Join(kl.getPodVolumeDevicesDir(podUID), pluginName) +} + // getPodPluginsDir returns the full path to the per-pod data directory under // which plugins may store data for the specified pod. This directory may not // exist if the pod does not exist. diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 8a180747423..5e365aa54a8 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -64,6 +64,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" utilfile "k8s.io/kubernetes/pkg/util/file" "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" volumevalidation "k8s.io/kubernetes/pkg/volume/validation" "k8s.io/kubernetes/third_party/forked/golang/expansion" @@ -91,9 +92,9 @@ func (kl *Kubelet) GetActivePods() []*v1.Pod { return activePods } -// makeDevices determines the devices for the given container. +// makeGPUDevices determines the devices for the given container. // Experimental. -func (kl *Kubelet) makeDevices(pod *v1.Pod, container *v1.Container) ([]kubecontainer.DeviceInfo, error) { +func (kl *Kubelet) makeGPUDevices(pod *v1.Pod, container *v1.Container) ([]kubecontainer.DeviceInfo, error) { if container.Resources.Limits.NvidiaGPU().IsZero() { return nil, nil } @@ -128,6 +129,39 @@ func makeAbsolutePath(goos, path string) string { return "c:\\" + path } +// makeBlockVolumes maps the raw block devices specified in the path of the container +// Experimental +func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVolumes kubecontainer.VolumeMap, blkutil volumeutil.BlockVolumePathHandler) ([]kubecontainer.DeviceInfo, error) { + var devices []kubecontainer.DeviceInfo + for _, device := range container.VolumeDevices { + // check path is absolute + if !filepath.IsAbs(device.DevicePath) { + return nil, fmt.Errorf("error DevicePath `%s` must be an absolute path", device.DevicePath) + } + vol, ok := podVolumes[device.Name] + if !ok || vol.BlockVolumeMapper == nil { + glog.Errorf("Block volume cannot be satisfied for container %q, because the volume is missing or the volume mapper is nil: %+v", container.Name, device) + return nil, fmt.Errorf("cannot find volume %q to pass into container %q", device.Name, container.Name) + } + // Get a symbolic link associated to a block device under pod device path + dirPath, volName := vol.BlockVolumeMapper.GetPodDeviceMapPath() + symlinkPath := path.Join(dirPath, volName) + if islinkExist, checkErr := blkutil.IsSymlinkExist(symlinkPath); checkErr != nil { + return nil, checkErr + } else if islinkExist { + // Check readOnly in PVCVolumeSource and set read only permission if it's true. + permission := "mrw" + if vol.ReadOnly { + permission = "r" + } + glog.V(4).Infof("Device will be attached to container %q. Path on host: %v", container.Name, symlinkPath) + devices = append(devices, kubecontainer.DeviceInfo{PathOnHost: symlinkPath, PathInContainer: device.DevicePath, Permissions: permission}) + } + } + + return devices, nil +} + // makeMounts determines the mount points for the given container. func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) { // Kubernetes only mounts on /etc/hosts if: @@ -416,12 +450,22 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai opts.PortMappings = kubecontainer.MakePortMappings(container) // TODO(random-liu): Move following convert functions into pkg/kubelet/container - devices, err := kl.makeDevices(pod, container) + devices, err := kl.makeGPUDevices(pod, container) if err != nil { return nil, err } opts.Devices = append(opts.Devices, devices...) + // TODO: remove feature gate check after no longer needed + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + blkutil := volumeutil.NewBlockVolumePathHandler() + blkVolumes, err := kl.makeBlockVolumes(pod, container, volumes, blkutil) + if err != nil { + return nil, err + } + opts.Devices = append(opts.Devices, blkVolumes...) + } + mounts, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) if err != nil { return nil, err diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 79e1f15361a..f313f80e056 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -47,6 +47,7 @@ import ( containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/server/portforward" "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" + volumetest "k8s.io/kubernetes/pkg/volume/testing" ) func TestMakeAbsolutePath(t *testing.T) { @@ -352,6 +353,139 @@ func TestMakeMounts(t *testing.T) { } } +func TestMakeBlockVolumes(t *testing.T) { + testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) + defer testKubelet.Cleanup() + kubelet := testKubelet.kubelet + testCases := map[string]struct { + container v1.Container + podVolumes kubecontainer.VolumeMap + expectErr bool + expectedErrMsg string + expectedDevices []kubecontainer.DeviceInfo + }{ + "valid volumeDevices in container": { + podVolumes: kubecontainer.VolumeMap{ + "disk1": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/", volName: "sda"}}, + "disk2": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/disk/by-path/", volName: "diskPath"}, ReadOnly: true}, + "disk3": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/disk/by-id/", volName: "diskUuid"}}, + "disk4": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/var/lib/", volName: "rawdisk"}, ReadOnly: true}, + }, + container: v1.Container{ + Name: "container1", + VolumeDevices: []v1.VolumeDevice{ + { + DevicePath: "/dev/sda", + Name: "disk1", + }, + { + DevicePath: "/dev/xvda", + Name: "disk2", + }, + { + DevicePath: "/dev/xvdb", + Name: "disk3", + }, + { + DevicePath: "/mnt/rawdisk", + Name: "disk4", + }, + }, + }, + expectedDevices: []kubecontainer.DeviceInfo{ + { + PathInContainer: "/dev/sda", + PathOnHost: "/dev/sda", + Permissions: "mrw", + }, + { + PathInContainer: "/dev/xvda", + PathOnHost: "/dev/disk/by-path/diskPath", + Permissions: "r", + }, + { + PathInContainer: "/dev/xvdb", + PathOnHost: "/dev/disk/by-id/diskUuid", + Permissions: "mrw", + }, + { + PathInContainer: "/mnt/rawdisk", + PathOnHost: "/var/lib/rawdisk", + Permissions: "r", + }, + }, + expectErr: false, + }, + "invalid absolute Path": { + podVolumes: kubecontainer.VolumeMap{ + "disk": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/", volName: "sda"}}, + }, + container: v1.Container{ + VolumeDevices: []v1.VolumeDevice{ + { + DevicePath: "must/be/absolute", + Name: "disk", + }, + }, + }, + expectErr: true, + expectedErrMsg: "error DevicePath `must/be/absolute` must be an absolute path", + }, + "volume doesn't exist": { + podVolumes: kubecontainer.VolumeMap{}, + container: v1.Container{ + VolumeDevices: []v1.VolumeDevice{ + { + DevicePath: "/dev/sdaa", + Name: "disk", + }, + }, + }, + expectErr: true, + expectedErrMsg: "cannot find volume \"disk\" to pass into container \"\"", + }, + "volume BlockVolumeMapper is nil": { + podVolumes: kubecontainer.VolumeMap{ + "disk": kubecontainer.VolumeInfo{}, + }, + container: v1.Container{ + VolumeDevices: []v1.VolumeDevice{ + { + DevicePath: "/dev/sdzz", + Name: "disk", + }, + }, + }, + expectErr: true, + expectedErrMsg: "cannot find volume \"disk\" to pass into container \"\"", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + pod := v1.Pod{ + Spec: v1.PodSpec{ + HostNetwork: true, + }, + } + blkutil := volumetest.NewBlockVolumePathHandler() + blkVolumes, err := kubelet.makeBlockVolumes(&pod, &tc.container, tc.podVolumes, blkutil) + // validate only the error if we expect an error + if tc.expectErr { + if err == nil || err.Error() != tc.expectedErrMsg { + t.Fatalf("expected error message `%s` but got `%v`", tc.expectedErrMsg, err) + } + return + } + // otherwise validate the devices + if err != nil { + t.Fatal(err) + } + assert.Equal(t, tc.expectedDevices, blkVolumes, "devices of container %+v", tc.container) + }) + } +} + func TestNodeHostsFileContent(t *testing.T) { testCases := []struct { hostsFileName string diff --git a/pkg/kubelet/kubelet_volumes_test.go b/pkg/kubelet/kubelet_volumes_test.go index c3b2d53c290..60f1e70287d 100644 --- a/pkg/kubelet/kubelet_volumes_test.go +++ b/pkg/kubelet/kubelet_volumes_test.go @@ -448,3 +448,23 @@ func (f *stubVolume) SetUp(fsGroup *int64) error { func (f *stubVolume) SetUpAt(dir string, fsGroup *int64) error { return nil } + +type stubBlockVolume struct { + dirPath string + volName string +} + +func (f *stubBlockVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) { + return "", nil +} + +func (f *stubBlockVolume) GetPodDeviceMapPath() (string, string) { + return f.dirPath, f.volName +} + +func (f *stubBlockVolume) SetUpDevice() (string, error) { + return "", nil +} +func (f *stubBlockVolume) TearDownDevice(mapPath string, devicePath string) error { + return nil +} diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 25194512647..c0baa1cf57b 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -86,10 +86,18 @@ type kubeletVolumeHost struct { mountPodManager mountpod.Manager } +func (kvh *kubeletVolumeHost) GetVolumeDevicePluginDir(pluginName string) string { + return kvh.kubelet.getVolumeDevicePluginDir(pluginName) +} + func (kvh *kubeletVolumeHost) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string { return kvh.kubelet.getPodVolumeDir(podUID, pluginName, volumeName) } +func (kvh *kubeletVolumeHost) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string { + return kvh.kubelet.getPodVolumeDeviceDir(podUID, pluginName) +} + func (kvh *kubeletVolumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string { return kvh.kubelet.getPodPluginDir(podUID, pluginName) } diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index ff3b8925ca3..c22049e7bca 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -58,7 +58,7 @@ type ActualStateOfWorld interface { // volume, reset the pod's remountRequired value. // If a volume with the name volumeName does not exist in the list of // attached volumes, an error is returned. - AddPodToVolume(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, outerVolumeSpecName string, volumeGidValue string) error + AddPodToVolume(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error // MarkRemountRequired marks each volume that is successfully attached and // mounted for the specified pod as requiring remount (if the plugin for the @@ -254,6 +254,9 @@ type mountedPod struct { // mounter used to mount mounter volume.Mounter + // mapper used to block volumes support + blockVolumeMapper volume.BlockVolumeMapper + // outerVolumeSpecName is the volume.Spec.Name() of the volume as referenced // directly in the pod. If the volume was referenced through a persistent // volume claim, this contains the volume.Spec.Name() of the persistent @@ -287,6 +290,7 @@ func (asw *actualStateOfWorld) MarkVolumeAsMounted( podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, + blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error { return asw.AddPodToVolume( @@ -294,6 +298,7 @@ func (asw *actualStateOfWorld) MarkVolumeAsMounted( podUID, volumeName, mounter, + blockVolumeMapper, outerVolumeSpecName, volumeGidValue) } @@ -385,6 +390,7 @@ func (asw *actualStateOfWorld) AddPodToVolume( podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, + blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error { asw.Lock() @@ -403,6 +409,7 @@ func (asw *actualStateOfWorld) AddPodToVolume( podName: podName, podUID: podUID, mounter: mounter, + blockVolumeMapper: blockVolumeMapper, outerVolumeSpecName: outerVolumeSpecName, volumeGidValue: volumeGidValue, } @@ -682,5 +689,7 @@ func getMountedVolume( PluginName: attachedVolume.pluginName, PodUID: mountedPod.podUID, Mounter: mountedPod.mounter, - VolumeGidValue: mountedPod.volumeGidValue}} + BlockVolumeMapper: mountedPod.blockVolumeMapper, + VolumeGidValue: mountedPod.volumeGidValue, + VolumeSpec: attachedVolume.spec}} } diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go index 376be8461f2..e2c51812d44 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go @@ -204,9 +204,14 @@ func Test_AddPodToVolume_Positive_ExistingVolumeNewNode(t *testing.T) { t.Fatalf("NewMounter failed. Expected: Actual: <%v>", err) } + mapper, err := plugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("NewBlockVolumeMapper failed. Expected: Actual: <%v>", err) + } + // Act err = asw.AddPodToVolume( - podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */) // Assert if err != nil { @@ -263,15 +268,20 @@ func Test_AddPodToVolume_Positive_ExistingVolumeExistingNode(t *testing.T) { t.Fatalf("NewMounter failed. Expected: Actual: <%v>", err) } + mapper, err := plugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("NewBlockVolumeMapper failed. Expected: Actual: <%v>", err) + } + err = asw.AddPodToVolume( - podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } // Act err = asw.AddPodToVolume( - podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */) // Assert if err != nil { @@ -318,6 +328,15 @@ func Test_AddPodToVolume_Negative_VolumeDoesntExist(t *testing.T) { volumeSpec, err) } + + blockplugin, err := volumePluginMgr.FindMapperPluginBySpec(volumeSpec) + if err != nil { + t.Fatalf( + "volumePluginMgr.FindMapperPluginBySpec failed to find volume plugin for %#v with: %v", + volumeSpec, + err) + } + volumeName, err := volumehelper.GetUniqueVolumeNameFromSpec( plugin, volumeSpec) @@ -328,9 +347,14 @@ func Test_AddPodToVolume_Negative_VolumeDoesntExist(t *testing.T) { t.Fatalf("NewMounter failed. Expected: Actual: <%v>", err) } + mapper, err := blockplugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("NewBlockVolumeMapper failed. Expected: Actual: <%v>", err) + } + // Act err = asw.AddPodToVolume( - podName, pod.UID, volumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod.UID, volumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */) // Assert if err == nil { diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go index 4c457217c6e..a9ecaaad69e 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go @@ -31,7 +31,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/pod" @@ -260,11 +262,12 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes(pod *v1.Pod) { } allVolumesAdded := true + mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers) // Process volume spec for each volume defined in pod for _, podVolume := range pod.Spec.Volumes { volumeSpec, volumeGidValue, err := - dswp.createVolumeSpec(podVolume, pod.Namespace) + dswp.createVolumeSpec(podVolume, pod.Name, pod.Namespace, mountsMap, devicesMap) if err != nil { glog.Errorf( "Error processing volume %q for pod %q: %v", @@ -336,7 +339,7 @@ func (dswp *desiredStateOfWorldPopulator) deleteProcessedPod( // specified volume. It dereference any PVC to get PV objects, if needed. // Returns an error if unable to obtain the volume at this time. func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( - podVolume v1.Volume, podNamespace string) (*volume.Spec, string, error) { + podVolume v1.Volume, podName string, podNamespace string, mountsMap map[string]bool, devicesMap map[string]bool) (*volume.Spec, string, error) { if pvcSource := podVolume.VolumeSource.PersistentVolumeClaim; pvcSource != nil { glog.V(10).Infof( @@ -381,6 +384,31 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( pvcSource.ClaimName, pvcUID) + // TODO: remove feature gate check after no longer needed + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + volumeMode, err := volumehelper.GetVolumeMode(volumeSpec) + if err != nil { + return nil, "", err + } + // Error if a container has volumeMounts but the volumeMode of PVC isn't Filesystem + if mountsMap[podVolume.Name] && volumeMode != v1.PersistentVolumeFilesystem { + return nil, "", fmt.Errorf( + "Volume %q has volumeMode %q, but is specified in volumeMounts for pod %q/%q", + podVolume.Name, + volumeMode, + podNamespace, + podName) + } + // Error if a container has volumeDevices but the volumeMode of PVC isn't Block + if devicesMap[podVolume.Name] && volumeMode != v1.PersistentVolumeBlock { + return nil, "", fmt.Errorf( + "Volume %q has volumeMode %q, but is specified in volumeDevices for pod %q/%q", + podVolume.Name, + volumeMode, + podNamespace, + podName) + } + } return volumeSpec, volumeGidValue, nil } @@ -449,6 +477,28 @@ func (dswp *desiredStateOfWorldPopulator) getPVSpec( return volume.NewSpecFromPersistentVolume(pv, pvcReadOnly), volumeGidValue, nil } +func (dswp *desiredStateOfWorldPopulator) makeVolumeMap(containers []v1.Container) (map[string]bool, map[string]bool) { + volumeDevicesMap := make(map[string]bool) + volumeMountsMap := make(map[string]bool) + + for _, container := range containers { + if container.VolumeMounts != nil { + for _, mount := range container.VolumeMounts { + volumeMountsMap[mount.Name] = true + } + } + // TODO: remove feature gate check after no longer needed + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) && + container.VolumeDevices != nil { + for _, device := range container.VolumeDevices { + volumeDevicesMap[device.Name] = true + } + } + } + + return volumeMountsMap, volumeDevicesMap +} + func getPVVolumeGidAnnotationValue(pv *v1.PersistentVolume) string { if volumeGid, ok := pv.Annotations[volumehelper.VolumeGidAnnotationKey]; ok { return volumeGid diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index 94a65f2b41f..e9f27af810b 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -22,7 +22,10 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/kubelet/configmap" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" @@ -37,54 +40,37 @@ import ( ) func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { - fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) - fakeClient := &fake.Clientset{} - - fakeSecretManager := secret.NewFakeManager() - fakeConfigMapManager := configmap.NewFakeManager() - fakePodManager := kubepod.NewBasicPodManager( - podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager) - - fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr) - fakeRuntime := &containertest.FakeRuntime{} - - fakeStatusManager := status.NewManager(fakeClient, fakePodManager, &statustest.FakePodDeletionSafetyProvider{}) - - dswp := &desiredStateOfWorldPopulator{ - kubeClient: fakeClient, - loopSleepDuration: 100 * time.Millisecond, - getPodStatusRetryDuration: 2 * time.Second, - podManager: fakePodManager, - podStatusProvider: fakeStatusManager, - desiredStateOfWorld: fakesDSW, - pods: processedPods{ - processedPods: make(map[types.UniquePodName]bool)}, - kubeContainerRuntime: fakeRuntime, - keepTerminatedPodVolumes: false, - } - - pod := &v1.Pod{ + // create dswp + pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ - Name: "dswp-test-pod", - UID: "dswp-test-pod-uid", - Namespace: "dswp-test", + Name: "dswp-test-volume-name", }, - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, fakesDSW := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ { - Name: "dswp-test-volume-name", - VolumeSource: v1.VolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: "dswp-test-fake-device", - }, - }, + Name: "dswp-test-volume-name", + MountPath: "/mnt", }, }, }, - Status: v1.PodStatus{ - Phase: v1.PodPhase("Running"), - }, } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) @@ -158,6 +144,320 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { } +func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + + // create dswp + mode := v1.PersistentVolumeBlock + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"}, + VolumeMode: &mode, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, fakesDSW := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeDevices: []v1.VolumeDevice{ + { + Name: "dswp-test-volume-name", + DevicePath: "/dev/sdb", + }, + }, + }, + } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers) + + fakePodManager.AddPod(pod) + + podName := volumehelper.GetUniquePodName(pod) + + generatedVolumeName := "fake-plugin/" + pod.Spec.Volumes[0].Name + + dswp.findAndAddNewPods() + + if !dswp.pods.processedPods[podName] { + t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName) + } + + expectedVolumeName := v1.UniqueVolumeName(generatedVolumeName) + + volumeExists := fakesDSW.VolumeExists(expectedVolumeName) + if !volumeExists { + t.Fatalf( + "VolumeExists(%q) failed. Expected: Actual: <%v>", + expectedVolumeName, + volumeExists) + } + + if podExistsInVolume := fakesDSW.PodExistsInVolume( + podName, expectedVolumeName); !podExistsInVolume { + t.Fatalf( + "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", + podExistsInVolume) + } + + verifyVolumeExistsInVolumesToMount( + t, v1.UniqueVolumeName(generatedVolumeName), false /* expectReportedInUse */, fakesDSW) + + //let the pod be terminated + podGet, exist := fakePodManager.GetPodByName(pod.Namespace, pod.Name) + if !exist { + t.Fatalf("Failed to get pod by pod name: %s and namespace: %s", pod.Name, pod.Namespace) + } + podGet.Status.Phase = v1.PodFailed + + //pod is added to fakePodManager but fakeRuntime can not get the pod,so here findAndRemoveDeletedPods() will remove the pod and volumes it is mounted + dswp.findAndRemoveDeletedPods() + + if dswp.pods.processedPods[podName] { + t.Fatalf("Failed to remove pods from desired state of world since they no longer exist") + } + + volumeExists = fakesDSW.VolumeExists(expectedVolumeName) + if volumeExists { + t.Fatalf( + "VolumeExists(%q) failed. Expected: Actual: <%v>", + expectedVolumeName, + volumeExists) + } + + if podExistsInVolume := fakesDSW.PodExistsInVolume( + podName, expectedVolumeName); podExistsInVolume { + t.Fatalf( + "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", + podExistsInVolume) + } + + volumesToMount := fakesDSW.GetVolumesToMount() + for _, volume := range volumesToMount { + if volume.VolumeName == expectedVolumeName { + t.Fatalf( + "Found volume %v in the list of desired state of world volumes to mount. Expected not", + expectedVolumeName) + } + } + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +func TestCreateVolumeSpec_Valid_File_VolumeMounts(t *testing.T) { + // create dswp + mode := v1.PersistentVolumeFilesystem + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + VolumeMode: &mode, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "dswp-test-volume-name", + MountPath: "/mnt", + }, + }, + }, + } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) + + fakePodManager.AddPod(pod) + mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers) + volumeSpec, _, err := + dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap) + + // Assert + if volumeSpec == nil || err != nil { + t.Fatalf("Failed to create volumeSpec with combination of filesystem mode and volumeMounts. err: %v", err) + } +} + +func TestCreateVolumeSpec_Valid_Block_VolumeDevices(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + + // create dswp + mode := v1.PersistentVolumeBlock + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"}, + VolumeMode: &mode, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeDevices: []v1.VolumeDevice{ + { + Name: "dswp-test-volume-name", + DevicePath: "/dev/sdb", + }, + }, + }, + } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers) + + fakePodManager.AddPod(pod) + mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers) + volumeSpec, _, err := + dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap) + + // Assert + if volumeSpec == nil || err != nil { + t.Fatalf("Failed to create volumeSpec with combination of block mode and volumeDevices. err: %v", err) + } + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +func TestCreateVolumeSpec_Invalid_File_VolumeDevices(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + + // create dswp + mode := v1.PersistentVolumeFilesystem + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + VolumeMode: &mode, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeDevices: []v1.VolumeDevice{ + { + Name: "dswp-test-volume-name", + DevicePath: "/dev/sdb", + }, + }, + }, + } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) + + fakePodManager.AddPod(pod) + mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers) + volumeSpec, _, err := + dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap) + + // Assert + if volumeSpec != nil || err == nil { + t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted") + } + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +func TestCreateVolumeSpec_Invalid_Block_VolumeMounts(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + + // create dswp + mode := v1.PersistentVolumeBlock + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"}, + VolumeMode: &mode, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "dswp-test-volume-name", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) + + // create pod + containers := []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "dswp-test-volume-name", + MountPath: "/mnt", + }, + }, + }, + } + pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers) + + fakePodManager.AddPod(pod) + mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers) + volumeSpec, _, err := + dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap) + + // Assert + if volumeSpec != nil || err == nil { + t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted") + } + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + func verifyVolumeExistsInVolumesToMount(t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectReportedInUse bool, dsw cache.DesiredStateOfWorld) { volumesToMount := dsw.GetVolumesToMount() for _, volume := range volumesToMount { @@ -179,3 +479,67 @@ func verifyVolumeExistsInVolumesToMount(t *testing.T, expectedVolumeName v1.Uniq expectedVolumeName, volumesToMount) } + +func createPodWithVolume(pod, pv, pvc string, containers []v1.Container) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod, + UID: "dswp-test-pod-uid", + Namespace: "dswp-test", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: pv, + VolumeSource: v1.VolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "dswp-test-fake-device", + }, + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc, + }, + }, + }, + }, + Containers: containers, + }, + Status: v1.PodStatus{ + Phase: v1.PodPhase("Running"), + }, + } +} + +func createDswpWithVolume(t *testing.T, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) (*desiredStateOfWorldPopulator, kubepod.Manager, cache.DesiredStateOfWorld) { + fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) + fakeClient := &fake.Clientset{} + fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + return true, pvc, nil + }) + fakeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) { + return true, pv, nil + }) + + fakeSecretManager := secret.NewFakeManager() + fakeConfigMapManager := configmap.NewFakeManager() + fakePodManager := kubepod.NewBasicPodManager( + podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager) + + fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr) + fakeRuntime := &containertest.FakeRuntime{} + + fakeStatusManager := status.NewManager(fakeClient, fakePodManager, &statustest.FakePodDeletionSafetyProvider{}) + + dswp := &desiredStateOfWorldPopulator{ + kubeClient: fakeClient, + loopSleepDuration: 100 * time.Millisecond, + getPodStatusRetryDuration: 2 * time.Second, + podManager: fakePodManager, + podStatusProvider: fakeStatusManager, + desiredStateOfWorld: fakesDSW, + pods: processedPods{ + processedPods: make(map[types.UniquePodName]bool)}, + kubeContainerRuntime: fakeRuntime, + keepTerminatedPodVolumes: false, + } + return dswp, fakePodManager, fakesDSW +} diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 00e5f046f82..2cb07477c74 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -22,6 +22,7 @@ package reconciler import ( "fmt" "io/ioutil" + "os" "path" "time" @@ -30,13 +31,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" utilfile "k8s.io/kubernetes/pkg/util/file" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" "k8s.io/kubernetes/pkg/util/mount" - "k8s.io/kubernetes/pkg/util/strings" + utilstrings "k8s.io/kubernetes/pkg/util/strings" volumepkg "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -171,10 +174,12 @@ func (rc *reconciler) reconcile() { // Ensure volumes that should be unmounted are unmounted. for _, mountedVolume := range rc.actualStateOfWorld.GetMountedVolumes() { if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName) { - // Volume is mounted, unmount it - glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) - err := rc.operationExecutor.UnmountVolume( - mountedVolume.MountedVolume, rc.actualStateOfWorld) + volumeHandler, err := operationexecutor.NewVolumeHandler(mountedVolume.VolumeSpec, rc.operationExecutor) + if err != nil { + glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for UnmountVolume failed"), err).Error()) + continue + } + err = volumeHandler.UnmountVolumeHandler(mountedVolume.MountedVolume, rc.actualStateOfWorld) if err != nil && !nestedpendingoperations.IsAlreadyExists(err) && !exponentialbackoff.IsExponentialBackoff(err) { @@ -239,12 +244,12 @@ func (rc *reconciler) reconcile() { if isRemount { remountingLogStr = "Volume is already mounted to pod, but remount was requested." } - glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr)) - err := rc.operationExecutor.MountVolume( - rc.waitForAttachTimeout, - volumeToMount.VolumeToMount, - rc.actualStateOfWorld, - isRemount) + volumeHandler, err := operationexecutor.NewVolumeHandler(volumeToMount.VolumeSpec, rc.operationExecutor) + if err != nil { + glog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for MountVolume failed"), err).Error()) + continue + } + err = volumeHandler.MountVolumeHandler(rc.waitForAttachTimeout, volumeToMount.VolumeToMount, rc.actualStateOfWorld, isRemount, remountingLogStr) if err != nil && !nestedpendingoperations.IsAlreadyExists(err) && !exponentialbackoff.IsExponentialBackoff(err) { @@ -268,10 +273,12 @@ func (rc *reconciler) reconcile() { if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName) && !rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName) { if attachedVolume.GloballyMounted { - // Volume is globally mounted to device, unmount it - glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", "")) - err := rc.operationExecutor.UnmountDevice( - attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter) + volumeHandler, err := operationexecutor.NewVolumeHandler(attachedVolume.VolumeSpec, rc.operationExecutor) + if err != nil { + glog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for UnmountDevice failed"), err).Error()) + continue + } + err = volumeHandler.UnmountDeviceHandler(attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter) if err != nil && !nestedpendingoperations.IsAlreadyExists(err) && !exponentialbackoff.IsExponentialBackoff(err) { @@ -332,6 +339,7 @@ type podVolume struct { volumeSpecName string mountPath string pluginName string + volumeMode v1.PersistentVolumeMode } type reconstructedVolume struct { @@ -345,6 +353,7 @@ type reconstructedVolume struct { devicePath string reportedInUse bool mounter volumepkg.Mounter + blockVolumeMapper volumepkg.BlockVolumeMapper } // reconstructFromDisk scans the volume directories under the given pod directory. If the volume is not @@ -419,20 +428,44 @@ func (rc *reconciler) syncStates(podsDir string) { // Reconstruct Volume object and reconstructedVolume data structure by reading the pod's volume directories func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, error) { + // plugin initializations plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName) if err != nil { return nil, err } - volumeSpec, err := plugin.ConstructVolumeSpec(volume.volumeSpecName, volume.mountPath) + attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginByName(volume.pluginName) if err != nil { return nil, err } + + // Create volumeSpec pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ UID: types.UID(volume.podName), }, } - attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginByName(volume.pluginName) + // TODO: remove feature gate check after no longer needed + var mapperPlugin volumepkg.BlockVolumePlugin + tmpSpec := &volumepkg.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{}}} + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + mapperPlugin, err = rc.volumePluginMgr.FindMapperPluginByName(volume.pluginName) + if err != nil { + return nil, err + } + tmpSpec = &volumepkg.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{VolumeMode: &volume.volumeMode}}} + } + volumeHandler, err := operationexecutor.NewVolumeHandler(tmpSpec, rc.operationExecutor) + if err != nil { + return nil, err + } + volumeSpec, err := volumeHandler.ReconstructVolumeHandler( + plugin, + mapperPlugin, + pod.UID, + volume.podName, + volume.volumeSpecName, + volume.mountPath, + volume.pluginName) if err != nil { return nil, err } @@ -448,17 +481,14 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, uniqueVolumeName = volumehelper.GetUniqueVolumeNameForNonAttachableVolume(volume.podName, plugin, volumeSpec) } - if attachablePlugin != nil { - if isNotMount, mountCheckErr := rc.mounter.IsLikelyNotMountPoint(volume.mountPath); mountCheckErr != nil { - return nil, fmt.Errorf("Could not check whether the volume %q (spec.Name: %q) pod %q (UID: %q) is mounted with: %v", - uniqueVolumeName, - volumeSpec.Name(), - volume.podName, - pod.UID, - mountCheckErr) - } else if isNotMount { - return nil, fmt.Errorf("Volume: %q is not mounted", uniqueVolumeName) - } + // Check existence of mount point for filesystem volume or symbolic link for block volume + isExist, checkErr := volumeHandler.CheckVolumeExistence(volume.mountPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin) + if checkErr != nil { + return nil, err + } + // If mount or symlink doesn't exist, volume reconstruction should be failed + if !isExist { + return nil, fmt.Errorf("Volume: %q is not mounted", uniqueVolumeName) } volumeMounter, newMounterErr := plugin.NewMounter( @@ -467,7 +497,7 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, volumepkg.VolumeOptions{}) if newMounterErr != nil { return nil, fmt.Errorf( - "MountVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v", + "reconstructVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v", uniqueVolumeName, volumeSpec.Name(), volume.podName, @@ -475,6 +505,27 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, newMounterErr) } + // TODO: remove feature gate check after no longer needed + var volumeMapper volumepkg.BlockVolumeMapper + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + var newMapperErr error + if mapperPlugin != nil { + volumeMapper, newMapperErr = mapperPlugin.NewBlockVolumeMapper( + volumeSpec, + pod, + volumepkg.VolumeOptions{}) + if newMapperErr != nil { + return nil, fmt.Errorf( + "reconstructVolume.NewBlockVolumeMapper failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v", + uniqueVolumeName, + volumeSpec.Name(), + volume.podName, + pod.UID, + newMapperErr) + } + } + } + reconstructedVolume := &reconstructedVolume{ volumeName: uniqueVolumeName, podName: volume.podName, @@ -489,6 +540,7 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, volumeGidValue: "", devicePath: "", mounter: volumeMounter, + blockVolumeMapper: volumeMapper, } return reconstructedVolume, nil } @@ -518,7 +570,6 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re volumeToMount.VolumeName, volume.outerVolumeSpecName) } } - for _, volume := range volumesNeedUpdate { err := rc.actualStateOfWorld.MarkVolumeAsAttached( volume.volumeName, volume.volumeSpec, "" /* nodeName */, volume.devicePath) @@ -532,6 +583,7 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re types.UID(volume.podName), volume.volumeName, volume.mounter, + volume.blockVolumeMapper, volume.outerVolumeSpecName, volume.volumeGidValue) if err != nil { @@ -574,33 +626,45 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) { } podName := podsDirInfo[i].Name() podDir := path.Join(podDir, podName) - volumesDir := path.Join(podDir, config.DefaultKubeletVolumesDirName) - volumesDirInfo, err := ioutil.ReadDir(volumesDir) - if err != nil { - glog.Errorf("Could not read volume directory %q: %v", volumesDir, err) - continue - } - for _, volumeDir := range volumesDirInfo { - pluginName := volumeDir.Name() - volumePluginPath := path.Join(volumesDir, pluginName) - volumePluginDirs, err := utilfile.ReadDirNoStat(volumePluginPath) - if err != nil { - glog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err) + // Find filesystem volume information + // ex. filesystem volume: /pods/{podUid}/volume/{escapeQualifiedPluginName}/{volumeName} + volumesDirs := map[v1.PersistentVolumeMode]string{ + v1.PersistentVolumeFilesystem: path.Join(podDir, config.DefaultKubeletVolumesDirName), + } + // TODO: remove feature gate check after no longer needed + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + // Find block volume information + // ex. block volume: /pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName} + volumesDirs[v1.PersistentVolumeBlock] = path.Join(podDir, config.DefaultKubeletVolumeDevicesDirName) + } + for volumeMode, volumesDir := range volumesDirs { + var volumesDirInfo []os.FileInfo + if volumesDirInfo, err = ioutil.ReadDir(volumesDir); err != nil { + // Just skip the loop becuase given volumesDir doesn't exist depending on volumeMode continue } - - unescapePluginName := strings.UnescapeQualifiedNameForDisk(pluginName) - for _, volumeName := range volumePluginDirs { - mountPath := path.Join(volumePluginPath, volumeName) - volumes = append(volumes, podVolume{ - podName: volumetypes.UniquePodName(podName), - volumeSpecName: volumeName, - mountPath: mountPath, - pluginName: unescapePluginName, - }) + for _, volumeDir := range volumesDirInfo { + pluginName := volumeDir.Name() + volumePluginPath := path.Join(volumesDir, pluginName) + volumePluginDirs, err := utilfile.ReadDirNoStat(volumePluginPath) + if err != nil { + glog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err) + continue + } + unescapePluginName := utilstrings.UnescapeQualifiedNameForDisk(pluginName) + for _, volumeName := range volumePluginDirs { + mountPath := path.Join(volumePluginPath, volumeName) + glog.V(5).Infof("podName: %v, mount path from volume plugin directory: %v, ", podName, mountPath) + volumes = append(volumes, podVolume{ + podName: volumetypes.UniquePodName(podName), + volumeSpecName: volumeName, + mountPath: mountPath, + pluginName: unescapePluginName, + volumeMode: volumeMode, + }) + } } - } } glog.V(10).Infof("Get volumes from pod directory %q %+v", podDir, volumes) diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go index ca3b1d00071..32cd954d2ec 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go @@ -23,10 +23,12 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" @@ -61,7 +63,14 @@ func Test_Run_Positive_DoNothing(t *testing.T) { asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} - oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler, + )) reconciler := NewReconciler( kubeClient, false, /* controllerAttachDetachEnabled */ @@ -99,7 +108,13 @@ func Test_Run_Positive_VolumeAttachAndMount(t *testing.T) { asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} - oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) reconciler := NewReconciler( kubeClient, false, /* controllerAttachDetachEnabled */ @@ -171,7 +186,13 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabled(t *testing.T) { asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} - oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) reconciler := NewReconciler( kubeClient, true, /* controllerAttachDetachEnabled */ @@ -244,7 +265,13 @@ func Test_Run_Positive_VolumeAttachMountUnmountDetach(t *testing.T) { asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} - oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) reconciler := NewReconciler( kubeClient, false, /* controllerAttachDetachEnabled */ @@ -328,7 +355,13 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) { asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} - oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) reconciler := NewReconciler( kubeClient, true, /* controllerAttachDetachEnabled */ @@ -399,6 +432,517 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) { assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) } +// Populates desiredStateOfWorld cache with one volume/pod. +// Calls Run() +// Verifies there are attach/get map paths/setupDevice calls and +// no detach/teardownDevice calls. +func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + // Arrange + volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + kubeClient := createTestClient() + fakeRecorder := &record.FakeRecorder{} + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) + reconciler := NewReconciler( + kubeClient, + false, /* controllerAttachDetachEnabled */ + reconcilerLoopSleepDuration, + reconcilerSyncStatesSleepPeriod, + waitForAttachTimeout, + nodeName, + dsw, + asw, + hasAddedPods, + oex, + &mount.FakeMounter{}, + volumePluginMgr, + kubeletPodsDir) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{}, + } + + mode := v1.PersistentVolumeBlock + gcepv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"}, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")}, + PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}}, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + v1.ReadOnlyMany, + }, + VolumeMode: &mode, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: gcepv, + } + podName := volumehelper.GetUniquePodName(pod) + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + // Act + runReconciler(reconciler) + waitForMount(t, fakePlugin, generatedVolumeName, asw) + // Assert + assert.NoError(t, volumetesting.VerifyAttachCallCount( + 1 /* expectedAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount( + 1 /* expectedWaitForAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount( + 1 /* expectedGetGlobalMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount( + 1 /* expectedPodDeviceMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount( + 1 /* expectedSetUpDeviceCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +// Populates desiredStateOfWorld cache with one volume/pod. +// Enables controllerAttachDetachEnabled. +// Calls Run() +// Verifies there are two get map path calls, a setupDevice call +// and no teardownDevice call. +// Verifies there are no attach/detach calls. +func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + // Arrange + volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + kubeClient := createTestClient() + fakeRecorder := &record.FakeRecorder{} + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) + reconciler := NewReconciler( + kubeClient, + true, /* controllerAttachDetachEnabled */ + reconcilerLoopSleepDuration, + reconcilerSyncStatesSleepPeriod, + waitForAttachTimeout, + nodeName, + dsw, + asw, + hasAddedPods, + oex, + &mount.FakeMounter{}, + volumePluginMgr, + kubeletPodsDir) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{}, + } + + mode := v1.PersistentVolumeBlock + gcepv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"}, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")}, + PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}}, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + v1.ReadOnlyMany, + }, + VolumeMode: &mode, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: gcepv, + } + podName := volumehelper.GetUniquePodName(pod) + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + // Act + runReconciler(reconciler) + waitForMount(t, fakePlugin, generatedVolumeName, asw) + + // Assert + assert.NoError(t, volumetesting.VerifyZeroAttachCalls(fakePlugin)) + assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount( + 1 /* expectedWaitForAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount( + 1 /* expectedGetGlobalMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount( + 1 /* expectedPodDeviceMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount( + 1 /* expectedSetUpCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +// Populates desiredStateOfWorld cache with one volume/pod. +// Calls Run() +// Verifies there is one attach call, two get map path calls, +// setupDevice call and no detach calls. +// Deletes volume/pod from desired state of world. +// Verifies one detach/teardownDevice calls are issued. +func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + // Arrange + volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + kubeClient := createTestClient() + fakeRecorder := &record.FakeRecorder{} + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) + reconciler := NewReconciler( + kubeClient, + false, /* controllerAttachDetachEnabled */ + reconcilerLoopSleepDuration, + reconcilerSyncStatesSleepPeriod, + waitForAttachTimeout, + nodeName, + dsw, + asw, + hasAddedPods, + oex, + &mount.FakeMounter{}, + volumePluginMgr, + kubeletPodsDir) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{}, + } + + mode := v1.PersistentVolumeBlock + gcepv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"}, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")}, + PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}}, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + v1.ReadOnlyMany, + }, + VolumeMode: &mode, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: gcepv, + } + podName := volumehelper.GetUniquePodName(pod) + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + // Act + runReconciler(reconciler) + waitForMount(t, fakePlugin, generatedVolumeName, asw) + // Assert + assert.NoError(t, volumetesting.VerifyAttachCallCount( + 1 /* expectedAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount( + 1 /* expectedWaitForAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount( + 1 /* expectedGetGlobalMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount( + 1 /* expectedPodDeviceMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount( + 1 /* expectedSetUpCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) + + // Act + dsw.DeletePodFromVolume(podName, generatedVolumeName) + waitForDetach(t, fakePlugin, generatedVolumeName, asw) + + // Assert + assert.NoError(t, volumetesting.VerifyTearDownDeviceCallCount( + 1 /* expectedTearDownDeviceCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyDetachCallCount( + 1 /* expectedDetachCallCount */, fakePlugin)) + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +// Populates desiredStateOfWorld cache with one volume/pod. +// Enables controllerAttachDetachEnabled. +// Calls Run() +// Verifies two map path calls are made and no teardownDevice/detach calls. +// Deletes volume/pod from desired state of world. +// Verifies one teardownDevice call is made. +// Verifies there are no attach/detach calls made. +func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) { + // Enable BlockVolume feature gate + utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + // Arrange + volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + kubeClient := createTestClient() + fakeRecorder := &record.FakeRecorder{} + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) + reconciler := NewReconciler( + kubeClient, + true, /* controllerAttachDetachEnabled */ + reconcilerLoopSleepDuration, + reconcilerSyncStatesSleepPeriod, + waitForAttachTimeout, + nodeName, + dsw, + asw, + hasAddedPods, + oex, + &mount.FakeMounter{}, + volumePluginMgr, + kubeletPodsDir) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{}, + } + + mode := v1.PersistentVolumeBlock + gcepv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"}, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")}, + PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}}, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + v1.ReadOnlyMany, + }, + VolumeMode: &mode, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: gcepv, + } + podName := volumehelper.GetUniquePodName(pod) + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + // Act + runReconciler(reconciler) + + dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) + waitForMount(t, fakePlugin, generatedVolumeName, asw) + + // Assert + assert.NoError(t, volumetesting.VerifyZeroAttachCalls(fakePlugin)) + assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount( + 1 /* expectedWaitForAttachCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount( + 1 /* expectedGetGlobalMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount( + 1 /* expectedPodDeviceMapPathCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount( + 1 /* expectedSetUpCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) + + // Act + dsw.DeletePodFromVolume(podName, generatedVolumeName) + waitForDetach(t, fakePlugin, generatedVolumeName, asw) + + // Assert + assert.NoError(t, volumetesting.VerifyTearDownDeviceCallCount( + 1 /* expectedTearDownDeviceCallCount */, fakePlugin)) + assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) + + // Rollback feature gate to false. + utilfeature.DefaultFeatureGate.Set("BlockVolume=false") +} + +func Test_GenerateMapVolumeFunc_Plugin_Not_Found(t *testing.T) { + testCases := map[string]struct { + volumePlugins []volume.VolumePlugin + expectErr bool + expectedErrMsg string + }{ + "volumePlugin is nil": { + volumePlugins: []volume.VolumePlugin{}, + expectErr: true, + expectedErrMsg: "MapVolume.FindMapperPluginBySpec failed", + }, + "blockVolumePlugin is nil": { + volumePlugins: volumetesting.NewFakeFileVolumePlugin(), + expectErr: true, + expectedErrMsg: "MapVolume.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + volumePluginMgr := &volume.VolumePluginMgr{} + volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + nil, /* kubeClient */ + volumePluginMgr, + nil, /* fakeRecorder */ + false, /* checkNodeCapabilitiesBeforeMount */ + nil)) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{}, + } + volumeToMount := operationexecutor.VolumeToMount{Pod: pod, VolumeSpec: &volume.Spec{}} + err := oex.MapVolume(waitForAttachTimeout, volumeToMount, asw) + // Assert + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expectedErrMsg) + } + }) + } +} + +func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) { + testCases := map[string]struct { + volumePlugins []volume.VolumePlugin + expectErr bool + expectedErrMsg string + }{ + "volumePlugin is nil": { + volumePlugins: []volume.VolumePlugin{}, + expectErr: true, + expectedErrMsg: "UnmapVolume.FindMapperPluginByName failed", + }, + "blockVolumePlugin is nil": { + volumePlugins: volumetesting.NewFakeFileVolumePlugin(), + expectErr: true, + expectedErrMsg: "UnmapVolume.FindMapperPluginByName failed to find BlockVolumeMapper plugin. Volume plugin is nil.", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + volumePluginMgr := &volume.VolumePluginMgr{} + volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + nil, /* kubeClient */ + volumePluginMgr, + nil, /* fakeRecorder */ + false, /* checkNodeCapabilitiesBeforeMount */ + nil)) + volumeToUnmount := operationexecutor.MountedVolume{PluginName: "fake-file-plugin"} + err := oex.UnmapVolume(volumeToUnmount, asw) + // Assert + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expectedErrMsg) + } + }) + } +} + +func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) { + testCases := map[string]struct { + volumePlugins []volume.VolumePlugin + expectErr bool + expectedErrMsg string + }{ + "volumePlugin is nil": { + volumePlugins: []volume.VolumePlugin{}, + expectErr: true, + expectedErrMsg: "UnmapDevice.FindMapperPluginBySpec failed", + }, + "blockVolumePlugin is nil": { + volumePlugins: volumetesting.NewFakeFileVolumePlugin(), + expectErr: true, + expectedErrMsg: "UnmapDevice.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + volumePluginMgr := &volume.VolumePluginMgr{} + volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + nil, /* kubeClient */ + volumePluginMgr, + nil, /* fakeRecorder */ + false, /* checkNodeCapabilitiesBeforeMount */ + nil)) + var mounter mount.Interface + deviceToDetach := operationexecutor.AttachedVolume{VolumeSpec: &volume.Spec{}} + err := oex.UnmapDevice(deviceToDetach, asw, mounter) + // Assert + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expectedErrMsg) + } + }) + } +} + func waitForMount( t *testing.T, fakePlugin *volumetesting.FakeVolumePlugin, diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index d48266f52a3..d3e7711407b 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -41,6 +41,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" "k8s.io/kubernetes/pkg/volume/util/types" "k8s.io/kubernetes/pkg/volume/util/volumehelper" @@ -168,8 +169,8 @@ func NewVolumeManager( kubeClient, volumePluginMgr, recorder, - checkNodeCapabilitiesBeforeMount), - ), + checkNodeCapabilitiesBeforeMount, + util.NewBlockVolumePathHandler())), } vm.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator( @@ -253,7 +254,11 @@ func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan str func (vm *volumeManager) GetMountedVolumesForPod(podName types.UniquePodName) container.VolumeMap { podVolumes := make(container.VolumeMap) for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) { - podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{Mounter: mountedVolume.Mounter} + podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{ + Mounter: mountedVolume.Mounter, + BlockVolumeMapper: mountedVolume.BlockVolumeMapper, + ReadOnly: mountedVolume.VolumeSpec.ReadOnly, + } } return podVolumes } diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 1ec55d2392b..bbfd1874f00 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -208,6 +208,26 @@ type ExpandableVolumePlugin interface { RequiresFSResize() bool } +// BlockVolumePlugin is an extend interface of VolumePlugin and is used for block volumes support. +type BlockVolumePlugin interface { + VolumePlugin + // NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification. + // Ownership of the spec pointer in *not* transferred. + // - spec: The v1.Volume spec + // - pod: The enclosing pod + NewBlockVolumeMapper(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error) + // NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state. + // - name: The volume name, as per the v1.Volume spec. + // - podUID: The UID of the enclosing pod + NewBlockVolumeUnmapper(name string, podUID types.UID) (BlockVolumeUnmapper, error) + // ConstructBlockVolumeSpec constructs a volume spec based on the given + // podUID, volume name and a pod device map path. + // The spec may have incomplete information due to limited information + // from input. This function is used by volume manager to reconstruct + // volume spec by reading the volume directories from disk. + ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error) +} + // VolumeHost is an interface that plugins can use to access the kubelet. type VolumeHost interface { // GetPluginDir returns the absolute path to a directory under which @@ -216,6 +236,11 @@ type VolumeHost interface { // GetPodPluginDir(). GetPluginDir(pluginName string) string + // GetVolumeDevicePluginDir returns the absolute path to a directory + // under which a given plugin may store data. + // ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/ + GetVolumeDevicePluginDir(pluginName string) string + // GetPodVolumeDir returns the absolute path a directory which // represents the named volume under the named plugin for the given // pod. If the specified pod does not exist, the result of this call @@ -228,6 +253,13 @@ type VolumeHost interface { // directory might not actually exist on disk yet. GetPodPluginDir(podUID types.UID, pluginName string) string + // GetPodVolumeDeviceDir returns the absolute path a directory which + // represents the named plugin for the given pod. + // If the specified pod does not exist, the result of this call + // might not exist. + // ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/ + GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string + // GetKubeClient returns a client interface GetKubeClient() clientset.Interface @@ -675,6 +707,32 @@ func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVo return nil, nil } +// FindMapperPluginBySpec fetches a block volume plugin by spec. +func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + + if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { + return blockVolumePlugin, nil + } + return nil, nil +} + +// FindMapperPluginByName fetches a block volume plugin by name. +func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + + if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { + return blockVolumePlugin, nil + } + return nil, nil +} + // NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler // pod. By default, a recycler pod simply runs "rm -rf" on a volume and tests // for emptiness. Most attributes of the template will be correct for most diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 035896b66be..c940abdbe45 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -39,6 +39,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" . "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) @@ -81,10 +82,18 @@ func (f *fakeVolumeHost) GetPluginDir(podUID string) string { return path.Join(f.rootDir, "plugins", podUID) } +func (f *fakeVolumeHost) GetVolumeDevicePluginDir(podUID string) string { + return path.Join(f.rootDir, "plugins", podUID) +} + func (f *fakeVolumeHost) GetPodVolumeDir(podUID types.UID, pluginName, volumeName string) string { return path.Join(f.rootDir, "pods", string(podUID), "volumes", pluginName, volumeName) } +func (f *fakeVolumeHost) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string { + return path.Join(f.rootDir, "pods", string(podUID), "volumeDevices", pluginName) +} + func (f *fakeVolumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string { return path.Join(f.rootDir, "pods", string(podUID), "plugins", pluginName) } @@ -194,13 +203,16 @@ type FakeVolumePlugin struct { NewAttacherCallCount int NewDetacherCallCount int - Mounters []*FakeVolume - Unmounters []*FakeVolume - Attachers []*FakeVolume - Detachers []*FakeVolume + Mounters []*FakeVolume + Unmounters []*FakeVolume + Attachers []*FakeVolume + Detachers []*FakeVolume + BlockVolumeMappers []*FakeVolume + BlockVolumeUnmappers []*FakeVolume } var _ VolumePlugin = &FakeVolumePlugin{} +var _ BlockVolumePlugin = &FakeVolumePlugin{} var _ RecyclableVolumePlugin = &FakeVolumePlugin{} var _ DeletableVolumePlugin = &FakeVolumePlugin{} var _ ProvisionableVolumePlugin = &FakeVolumePlugin{} @@ -280,6 +292,44 @@ func (plugin *FakeVolumePlugin) GetUnmounters() (Unmounters []*FakeVolume) { return plugin.Unmounters } +// Block volume support +func (plugin *FakeVolumePlugin) NewBlockVolumeMapper(spec *Spec, pod *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error) { + plugin.Lock() + defer plugin.Unlock() + volume := plugin.getFakeVolume(&plugin.BlockVolumeMappers) + if pod != nil { + volume.PodUID = pod.UID + } + volume.VolName = spec.Name() + volume.Plugin = plugin + return volume, nil +} + +// Block volume support +func (plugin *FakeVolumePlugin) GetBlockVolumeMapper() (BlockVolumeMappers []*FakeVolume) { + plugin.RLock() + defer plugin.RUnlock() + return plugin.BlockVolumeMappers +} + +// Block volume support +func (plugin *FakeVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (BlockVolumeUnmapper, error) { + plugin.Lock() + defer plugin.Unlock() + volume := plugin.getFakeVolume(&plugin.BlockVolumeUnmappers) + volume.PodUID = podUID + volume.VolName = volName + volume.Plugin = plugin + return volume, nil +} + +// Block volume support +func (plugin *FakeVolumePlugin) GetBlockVolumeUnmapper() (BlockVolumeUnmappers []*FakeVolume) { + plugin.RLock() + defer plugin.RUnlock() + return plugin.BlockVolumeUnmappers +} + func (plugin *FakeVolumePlugin) NewAttacher() (Attacher, error) { plugin.Lock() defer plugin.Unlock() @@ -345,10 +395,66 @@ func (plugin *FakeVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string }, nil } +// Block volume support +func (plugin *FakeVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error) { + return &Spec{ + Volume: &v1.Volume{ + Name: volumeName, + }, + }, nil +} + func (plugin *FakeVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { return []string{}, nil } +type FakeFileVolumePlugin struct { +} + +func (plugin *FakeFileVolumePlugin) Init(host VolumeHost) error { + return nil +} + +func (plugin *FakeFileVolumePlugin) GetPluginName() string { + return "fake-file-plugin" +} + +func (plugin *FakeFileVolumePlugin) GetVolumeName(spec *Spec) (string, error) { + return "", nil +} + +func (plugin *FakeFileVolumePlugin) CanSupport(spec *Spec) bool { + return true +} + +func (plugin *FakeFileVolumePlugin) RequiresRemount() bool { + return false +} + +func (plugin *FakeFileVolumePlugin) SupportsMountOption() bool { + return false +} + +func (plugin *FakeFileVolumePlugin) SupportsBulkVolumeVerification() bool { + return false +} + +func (plugin *FakeFileVolumePlugin) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) { + return nil, nil +} + +func (plugin *FakeFileVolumePlugin) NewUnmounter(name string, podUID types.UID) (Unmounter, error) { + return nil, nil +} + +func (plugin *FakeFileVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) { + return nil, nil +} + +func NewFakeFileVolumePlugin() []VolumePlugin { + return []VolumePlugin{&FakeFileVolumePlugin{}} +} + type FakeVolume struct { sync.RWMutex PodUID types.UID @@ -364,6 +470,10 @@ type FakeVolume struct { MountDeviceCallCount int UnmountDeviceCallCount int GetDeviceMountPathCallCount int + SetUpDeviceCallCount int + TearDownDeviceCallCount int + GlobalMapPathCallCount int + PodDeviceMapPathCallCount int } func (_ *FakeVolume) GetAttributes() Attributes { @@ -422,6 +532,76 @@ func (fv *FakeVolume) TearDownAt(dir string) error { return os.RemoveAll(dir) } +// Block volume support +func (fv *FakeVolume) SetUpDevice() (string, error) { + fv.Lock() + defer fv.Unlock() + fv.SetUpDeviceCallCount++ + return "", nil +} + +// Block volume support +func (fv *FakeVolume) GetSetUpDeviceCallCount() int { + fv.RLock() + defer fv.RUnlock() + return fv.SetUpDeviceCallCount +} + +// Block volume support +func (fv *FakeVolume) GetGlobalMapPath(spec *Spec) (string, error) { + fv.RLock() + defer fv.RUnlock() + fv.GlobalMapPathCallCount++ + return fv.getGlobalMapPath() +} + +// Block volume support +func (fv *FakeVolume) getGlobalMapPath() (string, error) { + return path.Join(fv.Plugin.Host.GetVolumeDevicePluginDir(utilstrings.EscapeQualifiedNameForDisk(fv.Plugin.PluginName)), "pluginDependentPath"), nil +} + +// Block volume support +func (fv *FakeVolume) GetGlobalMapPathCallCount() int { + fv.RLock() + defer fv.RUnlock() + return fv.GlobalMapPathCallCount +} + +// Block volume support +func (fv *FakeVolume) GetPodDeviceMapPath() (string, string) { + fv.RLock() + defer fv.RUnlock() + fv.PodDeviceMapPathCallCount++ + return fv.getPodDeviceMapPath() +} + +// Block volume support +func (fv *FakeVolume) getPodDeviceMapPath() (string, string) { + return path.Join(fv.Plugin.Host.GetPodVolumeDeviceDir(fv.PodUID, utilstrings.EscapeQualifiedNameForDisk(fv.Plugin.PluginName))), fv.VolName +} + +// Block volume support +func (fv *FakeVolume) GetPodDeviceMapPathCallCount() int { + fv.RLock() + defer fv.RUnlock() + return fv.PodDeviceMapPathCallCount +} + +// Block volume support +func (fv *FakeVolume) TearDownDevice(mapPath string, devicePath string) error { + fv.Lock() + defer fv.Unlock() + fv.TearDownDeviceCallCount++ + return nil +} + +// Block volume support +func (fv *FakeVolume) GetTearDownDeviceCallCount() int { + fv.RLock() + defer fv.RUnlock() + return fv.TearDownDeviceCallCount +} + func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error) { fv.Lock() defer fv.Unlock() @@ -439,7 +619,7 @@ func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, fv.Lock() defer fv.Unlock() fv.WaitForAttachCallCount++ - return "", nil + return "/dev/sdb", nil } func (fv *FakeVolume) GetWaitForAttachCallCount() int { @@ -540,6 +720,62 @@ func (fc *FakeProvisioner) Provision() (*v1.PersistentVolume, error) { return pv, nil } +var _ util.BlockVolumePathHandler = &FakeVolumePathHandler{} + +//NewDeviceHandler Create a new IoHandler implementation +func NewBlockVolumePathHandler() util.BlockVolumePathHandler { + return &FakeVolumePathHandler{} +} + +type FakeVolumePathHandler struct { + sync.RWMutex +} + +func (fv *FakeVolumePathHandler) MapDevice(devicePath string, mapDir string, linkName string) error { + // nil is success, else error + return nil +} + +func (fv *FakeVolumePathHandler) UnmapDevice(mapDir string, linkName string) error { + // nil is success, else error + return nil +} + +func (fv *FakeVolumePathHandler) RemoveMapPath(mapPath string) error { + // nil is success, else error + return nil +} + +func (fv *FakeVolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) { + // nil is success, else error + return true, nil +} + +func (fv *FakeVolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) { + // nil is success, else error + return []string{}, nil +} + +func (fv *FakeVolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) { + // nil is success, else error + return "", nil +} + +func (fv *FakeVolumePathHandler) AttachFileDevice(path string) (string, error) { + // nil is success, else error + return "", nil +} + +func (fv *FakeVolumePathHandler) GetLoopDevice(path string) (string, error) { + // nil is success, else error + return "/dev/loop1", nil +} + +func (fv *FakeVolumePathHandler) RemoveLoopDevice(device string) error { + // nil is success, else error + return nil +} + // FindEmptyDirectoryUsageOnTmpfs finds the expected usage of an empty directory existing on // a tmpfs filesystem on this system. func FindEmptyDirectoryUsageOnTmpfs() (*resource.Quantity, error) { @@ -758,6 +994,93 @@ func VerifyZeroDetachCallCount(fakeVolumePlugin *FakeVolumePlugin) error { return nil } +// VerifySetUpDeviceCallCount ensures that at least one of the Mappers for this +// plugin has the expectedSetUpDeviceCallCount number of calls. Otherwise it +// returns an error. +func VerifySetUpDeviceCallCount( + expectedSetUpDeviceCallCount int, + fakeVolumePlugin *FakeVolumePlugin) error { + for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() { + actualCallCount := mapper.GetSetUpDeviceCallCount() + if actualCallCount >= expectedSetUpDeviceCallCount { + return nil + } + } + + return fmt.Errorf( + "No Mapper have expected SetUpDeviceCallCount. Expected: <%v>.", + expectedSetUpDeviceCallCount) +} + +// VerifyTearDownDeviceCallCount ensures that at least one of the Unmappers for this +// plugin has the expectedTearDownDeviceCallCount number of calls. Otherwise it +// returns an error. +func VerifyTearDownDeviceCallCount( + expectedTearDownDeviceCallCount int, + fakeVolumePlugin *FakeVolumePlugin) error { + for _, unmapper := range fakeVolumePlugin.GetBlockVolumeUnmapper() { + actualCallCount := unmapper.GetTearDownDeviceCallCount() + if actualCallCount >= expectedTearDownDeviceCallCount { + return nil + } + } + + return fmt.Errorf( + "No Unmapper have expected TearDownDeviceCallCount. Expected: <%v>.", + expectedTearDownDeviceCallCount) +} + +// VerifyZeroTearDownDeviceCallCount ensures that all Mappers for this plugin have a +// zero TearDownDeviceCallCount. Otherwise it returns an error. +func VerifyZeroTearDownDeviceCallCount(fakeVolumePlugin *FakeVolumePlugin) error { + for _, unmapper := range fakeVolumePlugin.GetBlockVolumeUnmapper() { + actualCallCount := unmapper.GetTearDownDeviceCallCount() + if actualCallCount != 0 { + return fmt.Errorf( + "At least one unmapper has non-zero TearDownDeviceCallCount: <%v>.", + actualCallCount) + } + } + + return nil +} + +// VerifyGetGlobalMapPathCallCount ensures that at least one of the Mappers for this +// plugin has the expectedGlobalMapPathCallCount number of calls. Otherwise it returns +// an error. +func VerifyGetGlobalMapPathCallCount( + expectedGlobalMapPathCallCount int, + fakeVolumePlugin *FakeVolumePlugin) error { + for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() { + actualCallCount := mapper.GetGlobalMapPathCallCount() + if actualCallCount == expectedGlobalMapPathCallCount { + return nil + } + } + + return fmt.Errorf( + "No Mappers have expected GetGlobalMapPathCallCount. Expected: <%v>.", + expectedGlobalMapPathCallCount) +} + +// VerifyGetPodDeviceMapPathCallCount ensures that at least one of the Mappers for this +// plugin has the expectedPodDeviceMapPathCallCount number of calls. Otherwise it returns +// an error. +func VerifyGetPodDeviceMapPathCallCount( + expectedPodDeviceMapPathCallCount int, + fakeVolumePlugin *FakeVolumePlugin) error { + for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() { + actualCallCount := mapper.GetPodDeviceMapPathCallCount() + if actualCallCount == expectedPodDeviceMapPathCallCount { + return nil + } + } + + return fmt.Errorf( + "No Mappers have expected GetPodDeviceMapPathCallCount. Expected: <%v>.", + expectedPodDeviceMapPathCallCount) +} + // GetTestVolumePluginMgr creates, initializes, and returns a test volume plugin // manager and fake volume plugin using a fake volume host. func GetTestVolumePluginMgr( diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 8b11358436a..543067be6b7 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -29,7 +29,9 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" expandcache "k8s.io/kubernetes/pkg/controller/volume/expand/cache" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -105,6 +107,28 @@ type OperationExecutor interface { // actual state of the world to reflect that. UnmountDevice(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error + // MapVolume is used when the volumeMode is 'Block'. + // This method creates a symbolic link to the volume from both the pod + // specified in volumeToMount and global map path. + // Specifically it will: + // * Wait for the device to finish attaching (for attachable volumes only). + // * Update actual state of world to reflect volume is globally mounted/mapped. + // * Map volume to global map path using symbolic link. + // * Map the volume to the pod device map path using symbolic link. + // * Update actual state of world to reflect volume is mounted/mapped to the pod path. + MapVolume(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater) error + + // UnmapVolume unmaps symbolic link to the volume from both the pod device + // map path in volumeToUnmount and global map path. + // And then, updates the actual state of the world to reflect that. + UnmapVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error + + // UnmapDevice checks number of symbolic links under global map path. + // If number of reference is zero, remove global map path directory and + // free a volume for detach. + // It then updates the actual state of the world to reflect that. + UnmapDevice(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error + // VerifyControllerAttachedVolume checks if the specified volume is present // in the specified nodes AttachedVolumes Status field. It uses kubeClient // to fetch the node object. @@ -139,7 +163,7 @@ func NewOperationExecutor( // state of the world cache after successful mount/unmount. type ActualStateOfWorldMounterUpdater interface { // Marks the specified volume as mounted to the specified pod - MarkVolumeAsMounted(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, outerVolumeSpecName string, volumeGidValue string) error + MarkVolumeAsMounted(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error // Marks the specified volume as unmounted from the specified pod MarkVolumeAsUnmounted(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName) error @@ -495,8 +519,16 @@ type MountedVolume struct { // by kubelet to create container.VolumeMap. Mounter volume.Mounter + // BlockVolumeMapper is the volume mapper used to map this volume. It is required + // by kubelet to create container.VolumeMap. + BlockVolumeMapper volume.BlockVolumeMapper + // VolumeGidValue contains the value of the GID annotation, if present. VolumeGidValue string + + // VolumeSpec is a volume spec containing the specification for the volume + // that should be mounted. + VolumeSpec *volume.Spec } // GenerateMsgDetailed returns detailed msgs for mounted volumes @@ -733,6 +765,68 @@ func (oe *operationExecutor) ExpandVolume(pvcWithResizeRequest *expandcache.PVCW return oe.pendingOperations.Run(uniqueVolumeKey, "", expandFunc, opCompleteFunc) } +func (oe *operationExecutor) MapVolume( + waitForAttachTimeout time.Duration, + volumeToMount VolumeToMount, + actualStateOfWorld ActualStateOfWorldMounterUpdater) error { + mapFunc, plugin, err := oe.operationGenerator.GenerateMapVolumeFunc( + waitForAttachTimeout, volumeToMount, actualStateOfWorld) + if err != nil { + return err + } + + // Avoid executing map from multiple pods referencing the + // same volume in parallel + podName := nestedpendingoperations.EmptyUniquePodName + // TODO: remove this -- not necessary + if !volumeToMount.PluginIsAttachable { + // Non-attachable volume plugins can execute mount for multiple pods + // referencing the same volume in parallel + podName = volumehelper.GetUniquePodName(volumeToMount.Pod) + } + + opCompleteFunc := util.OperationCompleteHook(plugin, "map_volume") + return oe.pendingOperations.Run( + volumeToMount.VolumeName, podName, mapFunc, opCompleteFunc) +} + +func (oe *operationExecutor) UnmapVolume( + volumeToUnmount MountedVolume, + actualStateOfWorld ActualStateOfWorldMounterUpdater) error { + unmapFunc, plugin, err := + oe.operationGenerator.GenerateUnmapVolumeFunc(volumeToUnmount, actualStateOfWorld) + if err != nil { + return err + } + + // All volume plugins can execute unmap for multiple pods referencing the + // same volume in parallel + podName := volumetypes.UniquePodName(volumeToUnmount.PodUID) + + opCompleteFunc := util.OperationCompleteHook(plugin, "unmap_volume") + return oe.pendingOperations.Run( + volumeToUnmount.VolumeName, podName, unmapFunc, opCompleteFunc) +} + +func (oe *operationExecutor) UnmapDevice( + deviceToDetach AttachedVolume, + actualStateOfWorld ActualStateOfWorldMounterUpdater, + mounter mount.Interface) error { + unmapDeviceFunc, plugin, err := + oe.operationGenerator.GenerateUnmapDeviceFunc(deviceToDetach, actualStateOfWorld, mounter) + if err != nil { + return err + } + + // Avoid executing unmap device from multiple pods referencing + // the same volume in parallel + podName := nestedpendingoperations.EmptyUniquePodName + + opCompleteFunc := util.OperationCompleteHook(plugin, "unmap_device") + return oe.pendingOperations.Run( + deviceToDetach.VolumeName, podName, unmapDeviceFunc, opCompleteFunc) +} + func (oe *operationExecutor) VerifyControllerAttachedVolume( volumeToMount VolumeToMount, nodeName types.NodeName, @@ -748,6 +842,200 @@ func (oe *operationExecutor) VerifyControllerAttachedVolume( volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc, opCompleteFunc) } +// VolumeStateHandler defines a set of operations for handling mount/unmount/detach/reconstruct volume-related operations +type VolumeStateHandler interface { + // Volume is attached, mount/map it + MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool, remountingLogStr string) error + // Volume is mounted/mapped, unmount/unmap it + UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error + // Volume is not referenced from pod, unmount/unmap and detach it + UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error + // Reconstruct volume from mount path + ReconstructVolumeHandler(plugin volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, mountPath string, pluginName string) (*volume.Spec, error) + // check mount path if volume still exists + CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, attachable volume.AttachableVolumePlugin) (bool, error) +} + +// NewVolumeHandler return a new instance of volumeHandler depens on a volumeMode +func NewVolumeHandler(volumeSpec *volume.Spec, oe OperationExecutor) (VolumeStateHandler, error) { + + // TODO: remove feature gate check after no longer needed + var volumeHandler VolumeStateHandler + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + volumeMode, err := volumehelper.GetVolumeMode(volumeSpec) + if err != nil { + return nil, err + } + if volumeMode == v1.PersistentVolumeFilesystem { + volumeHandler = NewFilesystemVolumeHandler(oe) + } else { + volumeHandler = NewBlockVolumeHandler(oe) + } + } else { + volumeHandler = NewFilesystemVolumeHandler(oe) + } + return volumeHandler, nil +} + +// NewFilesystemVolumeHandler returns a new instance of FilesystemVolumeHandler. +func NewFilesystemVolumeHandler(operationExecutor OperationExecutor) FilesystemVolumeHandler { + return FilesystemVolumeHandler{ + oe: operationExecutor} +} + +// NewBlockVolumeHandler returns a new instance of BlockVolumeHandler. +func NewBlockVolumeHandler(operationExecutor OperationExecutor) BlockVolumeHandler { + return BlockVolumeHandler{ + oe: operationExecutor} +} + +// FilesystemVolumeHandler is VolumeHandler for Filesystem volume +type FilesystemVolumeHandler struct { + oe OperationExecutor +} + +// BlockVolumeHandler is VolumeHandler for Block volume +type BlockVolumeHandler struct { + oe OperationExecutor +} + +// MountVolumeHandler mount/remount a volume when a volume is attached +// This method is handler for filesystem volume +func (f FilesystemVolumeHandler) MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool, remountingLogStr string) error { + glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr)) + err := f.oe.MountVolume( + waitForAttachTimeout, + volumeToMount, + actualStateOfWorld, + isRemount) + return err +} + +// UnmountVolumeHandler unmount a volume if a volume is mounted +// This method is handler for filesystem volume +func (f FilesystemVolumeHandler) UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error { + glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) + err := f.oe.UnmountVolume( + mountedVolume, + actualStateOfWorld) + return err +} + +// UnmountDeviceHandler unmount and detach a device if a volume isn't referenced +// This method is handler for filesystem volume +func (f FilesystemVolumeHandler) UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error { + glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", "")) + err := f.oe.UnmountDevice( + attachedVolume, + actualStateOfWorld, + mounter) + return err +} + +// ReconstructVolumeHandler create volumeSpec from mount path +// This method is handler for filesystem volume +func (f FilesystemVolumeHandler) ReconstructVolumeHandler(plugin volume.VolumePlugin, _ volume.BlockVolumePlugin, _ types.UID, _ volumetypes.UniquePodName, volumeSpecName string, mountPath string, _ string) (*volume.Spec, error) { + glog.V(12).Infof("Starting operationExecutor.ReconstructVolumepodName") + volumeSpec, err := plugin.ConstructVolumeSpec(volumeSpecName, mountPath) + if err != nil { + return nil, err + } + return volumeSpec, nil +} + +// CheckVolumeExistence checks mount path directory if volume still exists, return true if volume is there +// Also return true for non-attachable volume case without mount point check +// This method is handler for filesystem volume +func (f FilesystemVolumeHandler) CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, attachable volume.AttachableVolumePlugin) (bool, error) { + if attachable != nil { + var isNotMount bool + var mountCheckErr error + if isNotMount, mountCheckErr = mounter.IsLikelyNotMountPoint(mountPath); mountCheckErr != nil { + return false, fmt.Errorf("Could not check whether the volume %q (spec.Name: %q) pod %q (UID: %q) is mounted with: %v", + uniqueVolumeName, + volumeName, + podName, + podUID, + mountCheckErr) + } + return !isNotMount, nil + } + return true, nil +} + +// MountVolumeHandler creates a map to device if a volume is attached +// This method is handler for block volume +func (b BlockVolumeHandler) MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, _ bool, _ string) error { + glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MapVolume", "")) + err := b.oe.MapVolume( + waitForAttachTimeout, + volumeToMount, + actualStateOfWorld) + return err +} + +// UnmountVolumeHandler unmap a volume if a volume is mapped +// This method is handler for block volume +func (b BlockVolumeHandler) UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error { + glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmapVolume", "")) + err := b.oe.UnmapVolume( + mountedVolume, + actualStateOfWorld) + return err +} + +// UnmountDeviceHandler detach a device and remove loopback if a volume isn't referenced +// This method is handler for block volume +func (b BlockVolumeHandler) UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error { + glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmapDevice", "")) + err := b.oe.UnmapDevice( + attachedVolume, + actualStateOfWorld, + mounter) + return err +} + +// ReconstructVolumeHandler create volumeSpec from mount path +// This method is handler for block volume +func (b BlockVolumeHandler) ReconstructVolumeHandler(_ volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, mountPath string, pluginName string) (*volume.Spec, error) { + glog.V(12).Infof("Starting operationExecutor.ReconstructVolume") + if mapperPlugin == nil { + return nil, fmt.Errorf("Could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)", + pluginName, + volumeSpecName, + podName, + uid) + } + // mountPath contains volumeName on the path. In the case of block volume, {volumeName} is symbolic link + // corresponding to raw block device. + // ex. mountPath: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} + volumeSpec, err := mapperPlugin.ConstructBlockVolumeSpec(uid, volumeSpecName, mountPath) + if err != nil { + return nil, err + } + return volumeSpec, nil +} + +// CheckVolumeExistence checks mount path directory if volume still exists, then return +// true if volume is there. Either plugin is attachable or non-attachable, the plugin +// should have symbolic link associated to raw block device under pod device map +// if volume exists. +// This method is handler for block volume +func (b BlockVolumeHandler) CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, _ volume.AttachableVolumePlugin) (bool, error) { + blkutil := util.NewBlockVolumePathHandler() + var islinkExist bool + var checkErr error + if islinkExist, checkErr = blkutil.IsSymlinkExist(mountPath); checkErr != nil { + return false, fmt.Errorf("Could not check whether the block volume %q (spec.Name: %q) pod %q (UID: %q) is mapped to: %v", + uniqueVolumeName, + volumeName, + podName, + podUID, + checkErr) + } + return islinkExist, nil +} + // TODO: this is a workaround for the unmount device issue caused by gci mounter. // In GCI cluster, if gci mounter is used for mounting, the container started by mounter // script will cause additional mounts created in the container. Since these mounts are diff --git a/pkg/volume/util/operationexecutor/operation_executor_test.go b/pkg/volume/util/operationexecutor/operation_executor_test.go index 35b7064c0d1..3624c889156 100644 --- a/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -40,6 +40,10 @@ const ( numVolumesToDetach = 2 numVolumesToVerifyAttached = 2 numVolumesToVerifyControllerAttached = 2 + numVolumesToMap = 2 + numAttachableVolumesToUnmap = 2 + numNonAttachableVolumesToUnmap = 2 + numDevicesToUnmap = 2 ) var _ OperationGenerator = &fakeOperationGenerator{} @@ -228,6 +232,113 @@ func TestOperationExecutor_VerifyControllerAttachedVolumeConcurrently(t *testing } } +func TestOperationExecutor_MapVolume_ConcurrentMapForNonAttachablePlugins(t *testing.T) { + // Arrange + ch, quit, oe := setup() + volumesToMount := make([]VolumeToMount, numVolumesToMap) + secretName := "secret-volume" + volumeName := v1.UniqueVolumeName(secretName) + + // Act + for i := range volumesToMount { + podName := "pod-" + strconv.Itoa((i + 1)) + pod := getTestPodWithSecret(podName, secretName) + volumesToMount[i] = VolumeToMount{ + Pod: pod, + VolumeName: volumeName, + PluginIsAttachable: false, // this field determines whether the plugin is attachable + ReportedInUse: true, + } + oe.MapVolume(0 /* waitForAttachTimeOut */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */) + } + + // Assert + if !isOperationRunConcurrently(ch, quit, numVolumesToMap) { + t.Fatalf("Unable to start map operations in Concurrent for non-attachable volumes") + } +} + +func TestOperationExecutor_MapVolume_ConcurrentMapForAttachablePlugins(t *testing.T) { + // Arrange + ch, quit, oe := setup() + volumesToMount := make([]VolumeToMount, numVolumesToAttach) + pdName := "pd-volume" + volumeName := v1.UniqueVolumeName(pdName) + + // Act + for i := range volumesToMount { + podName := "pod-" + strconv.Itoa((i + 1)) + pod := getTestPodWithGCEPD(podName, pdName) + volumesToMount[i] = VolumeToMount{ + Pod: pod, + VolumeName: volumeName, + PluginIsAttachable: true, // this field determines whether the plugin is attachable + ReportedInUse: true, + } + oe.MapVolume(0 /* waitForAttachTimeout */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */) + } + + // Assert + if !isOperationRunSerially(ch, quit) { + t.Fatalf("Map operations should not start concurrently for attachable volumes") + } +} + +func TestOperationExecutor_UnmapVolume_ConcurrentUnmapForAllPlugins(t *testing.T) { + // Arrange + ch, quit, oe := setup() + volumesToUnmount := make([]MountedVolume, numAttachableVolumesToUnmap+numNonAttachableVolumesToUnmap) + pdName := "pd-volume" + secretName := "secret-volume" + + // Act + for i := 0; i < numNonAttachableVolumesToUnmap+numAttachableVolumesToUnmap; i++ { + podName := "pod-" + strconv.Itoa(i+1) + if i < numNonAttachableVolumesToUnmap { + pod := getTestPodWithSecret(podName, secretName) + volumesToUnmount[i] = MountedVolume{ + PodName: volumetypes.UniquePodName(podName), + VolumeName: v1.UniqueVolumeName(secretName), + PodUID: pod.UID, + } + } else { + pod := getTestPodWithGCEPD(podName, pdName) + volumesToUnmount[i] = MountedVolume{ + PodName: volumetypes.UniquePodName(podName), + VolumeName: v1.UniqueVolumeName(pdName), + PodUID: pod.UID, + } + } + oe.UnmapVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */) + } + + // Assert + if !isOperationRunConcurrently(ch, quit, numNonAttachableVolumesToUnmap+numAttachableVolumesToUnmap) { + t.Fatalf("Unable to start unmap operations concurrently for volume plugins") + } +} + +func TestOperationExecutor_UnmapDeviceConcurrently(t *testing.T) { + // Arrange + ch, quit, oe := setup() + attachedVolumes := make([]AttachedVolume, numDevicesToUnmap) + pdName := "pd-volume" + + // Act + for i := range attachedVolumes { + attachedVolumes[i] = AttachedVolume{ + VolumeName: v1.UniqueVolumeName(pdName), + NodeName: "node-name", + } + oe.UnmapDevice(attachedVolumes[i], nil /* actualStateOfWorldMounterUpdater */, nil /* mount.Interface */) + } + + // Assert + if !isOperationRunSerially(ch, quit) { + t.Fatalf("Unmap device operations should not start concurrently") + } +} + type fakeOperationGenerator struct { ch chan interface{} quit chan interface{} @@ -302,6 +413,27 @@ func (fopg *fakeOperationGenerator) GenerateBulkVolumeVerifyFunc( }, nil } +func (fopg *fakeOperationGenerator) GenerateMapVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater) (func() error, string, error) { + return func() error { + startOperationAndBlock(fopg.ch, fopg.quit) + return nil + }, "", nil +} + +func (fopg *fakeOperationGenerator) GenerateUnmapVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { + return func() error { + startOperationAndBlock(fopg.ch, fopg.quit) + return nil + }, "", nil +} + +func (fopg *fakeOperationGenerator) GenerateUnmapDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) { + return func() error { + startOperationAndBlock(fopg.ch, fopg.quit) + return nil + }, "", nil +} + func (fopg *fakeOperationGenerator) GetVolumePluginMgr() *volume.VolumePluginMgr { return nil } diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 223a28cc379..909b3aae745 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -18,6 +18,7 @@ package operationexecutor import ( "fmt" + "strings" "time" "github.com/golang/glog" @@ -55,19 +56,24 @@ type operationGenerator struct { // which verifies that the components (binaries, etc.) required to mount // the volume are available on the underlying node before attempting mount. checkNodeCapabilitiesBeforeMount bool + + // blkUtil provides volume path related operations for block volume + blkUtil util.BlockVolumePathHandler } // NewOperationGenerator is returns instance of operationGenerator func NewOperationGenerator(kubeClient clientset.Interface, volumePluginMgr *volume.VolumePluginMgr, recorder record.EventRecorder, - checkNodeCapabilitiesBeforeMount bool) OperationGenerator { + checkNodeCapabilitiesBeforeMount bool, + blkUtil util.BlockVolumePathHandler) OperationGenerator { return &operationGenerator{ kubeClient: kubeClient, volumePluginMgr: volumePluginMgr, recorder: recorder, checkNodeCapabilitiesBeforeMount: checkNodeCapabilitiesBeforeMount, + blkUtil: blkUtil, } } @@ -94,6 +100,15 @@ type OperationGenerator interface { // Generates the function needed to check if the attach_detach controller has attached the volume plugin GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) + // Generates the MapVolume function needed to perform the map of a volume plugin + GenerateMapVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater) (func() error, string, error) + + // Generates the UnmapVolume function needed to perform the unmap of a volume plugin + GenerateUnmapVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) + + // Generates the UnmapDevice function needed to perform the unmap of a device + GenerateUnmapDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) + // GetVolumePluginMgr returns volume plugin manager GetVolumePluginMgr() *volume.VolumePluginMgr @@ -434,7 +449,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( return volumeToMount.GenerateErrorDetailed("MountVolume.WaitForAttach failed", err) } - glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", "")) + glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) @@ -501,6 +516,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( volumeToMount.Pod.UID, volumeToMount.VolumeName, volumeMounter, + nil, volumeToMount.OuterVolumeSpecName, volumeToMount.VolumeGidValue) if markVolMountedErr != nil { @@ -604,19 +620,9 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( // use mounter.PathIsDevice to check if the path is a device, // if so use mounter.DeviceOpened to check if the device is in use anywhere // else on the system. Retry if it returns true. - isDevicePath, devicePathErr := mounter.PathIsDevice(deviceToDetach.DevicePath) - var deviceOpened bool - var deviceOpenedErr error - if !isDevicePath && devicePathErr == nil { - // not a device path or path doesn't exist - //TODO: refer to #36092 - glog.V(3).Infof("Not checking device path %s", deviceToDetach.DevicePath) - deviceOpened = false - } else { - deviceOpened, deviceOpenedErr = mounter.DeviceOpened(deviceToDetach.DevicePath) - if deviceOpenedErr != nil { - return deviceToDetach.GenerateErrorDetailed("UnmountDevice.DeviceOpened failed", deviceOpenedErr) - } + deviceOpened, deviceOpenedErr := isDeviceOpened(deviceToDetach, mounter) + if deviceOpenedErr != nil { + return deviceOpenedErr } // The device is still in use elsewhere. Caller will log and retry. if deviceOpened { @@ -639,6 +645,337 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( }, attachableVolumePlugin.GetPluginName(), nil } +// GenerateMapVolumeFunc marks volume as mounted based on following steps. +// If plugin is attachable, call WaitForAttach() and then mark the device +// as mounted. On next step, SetUpDevice is called without dependent of +// plugin type, but this method mainly is targeted for none attachable plugin. +// After setup is done, create symbolic links on both global map path and pod +// device map path. Once symbolic links are created, take fd lock by +// loopback for the device to avoid silent volume replacement. This lock +// will be realased once no one uses the device. +// If all steps are completed, the volume is marked as unmounted. +func (og *operationGenerator) GenerateMapVolumeFunc( + waitForAttachTimeout time.Duration, + volumeToMount VolumeToMount, + actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { + + // Get block volume mapper plugin + var blockVolumeMapper volume.BlockVolumeMapper + blockVolumePlugin, err := + og.volumePluginMgr.FindMapperPluginBySpec(volumeToMount.VolumeSpec) + if err != nil { + return nil, "", volumeToMount.GenerateErrorDetailed("MapVolume.FindMapperPluginBySpec failed", err) + } + if blockVolumePlugin == nil { + return nil, "", volumeToMount.GenerateErrorDetailed("MapVolume.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil) + } + affinityErr := checkNodeAffinity(og, volumeToMount, blockVolumePlugin) + if affinityErr != nil { + return nil, blockVolumePlugin.GetPluginName(), affinityErr + } + blockVolumeMapper, newMapperErr := blockVolumePlugin.NewBlockVolumeMapper( + volumeToMount.VolumeSpec, + volumeToMount.Pod, + volume.VolumeOptions{}) + if newMapperErr != nil { + eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.NewBlockVolumeMapper initialization failed", newMapperErr) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error()) + return nil, blockVolumePlugin.GetPluginName(), detailedErr + } + + // Get attacher, if possible + attachableVolumePlugin, _ := + og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec) + var volumeAttacher volume.Attacher + if attachableVolumePlugin != nil { + volumeAttacher, _ = attachableVolumePlugin.NewAttacher() + } + + return func() error { + var devicePath string + if volumeAttacher != nil { + // Wait for attachable volumes to finish attaching + glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + + devicePath, err = volumeAttacher.WaitForAttach( + volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout) + if err != nil { + // On failure, return error. Caller will log and retry. + return volumeToMount.GenerateErrorDetailed("MapVolume.WaitForAttach failed", err) + } + + glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + + // Update actual state of world to reflect volume is globally mounted + markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( + volumeToMount.VolumeName) + if markDeviceMappedErr != nil { + // On failure, return error. Caller will log and retry. + return volumeToMount.GenerateErrorDetailed("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr) + } + } + // A plugin doesn't have attacher also needs to map device to global map path with SetUpDevice() + pluginDevicePath, mapErr := blockVolumeMapper.SetUpDevice() + if mapErr != nil { + // On failure, return error. Caller will log and retry. + eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.SetUp failed", mapErr) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error()) + return detailedErr + } + // Update devicePath for none attachable plugin case + if len(devicePath) == 0 { + if len(pluginDevicePath) != 0 { + devicePath = pluginDevicePath + } else { + return volumeToMount.GenerateErrorDetailed("MapVolume failed", fmt.Errorf("Device path of the volume is empty")) + } + } + // Set up global map path under the given plugin directory using symbolic link + globalMapPath, err := + blockVolumeMapper.GetGlobalMapPath(volumeToMount.VolumeSpec) + if err != nil { + // On failure, return error. Caller will log and retry. + return volumeToMount.GenerateErrorDetailed("MapVolume.GetDeviceMountPath failed", err) + } + mapErr = og.blkUtil.MapDevice(devicePath, globalMapPath, string(volumeToMount.Pod.UID)) + if mapErr != nil { + // On failure, return error. Caller will log and retry. + eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MapDevice failed", mapErr) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error()) + return detailedErr + } + // Device mapping for global map path succeeded + simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("globalMapPath %q", globalMapPath)) + verbosity := glog.Level(4) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg) + glog.V(verbosity).Infof(detailedMsg) + + // Map device to pod device map path under the given pod directory using symbolic link + volumeMapPath, volName := blockVolumeMapper.GetPodDeviceMapPath() + mapErr = og.blkUtil.MapDevice(devicePath, volumeMapPath, volName) + if mapErr != nil { + // On failure, return error. Caller will log and retry. + eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MapDevice failed", mapErr) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error()) + return detailedErr + } + + // Take filedescriptor lock to keep a block device opened. Otherwise, there is a case + // that the block device is silently removed and attached another device with same name. + // Container runtime can't handler this problem. To avoid unexpected condition fd lock + // for the block device is required. + _, err = og.blkUtil.AttachFileDevice(devicePath) + if err != nil { + return volumeToMount.GenerateErrorDetailed("MapVolume.AttachFileDevice failed", err) + } + + // Device mapping for pod device map path succeeded + simpleMsg, detailedMsg = volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("volumeMapPath %q", volumeMapPath)) + verbosity = glog.Level(1) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg) + glog.V(verbosity).Infof(detailedMsg) + + // Update actual state of world + markVolMountedErr := actualStateOfWorld.MarkVolumeAsMounted( + volumeToMount.PodName, + volumeToMount.Pod.UID, + volumeToMount.VolumeName, + nil, + blockVolumeMapper, + volumeToMount.OuterVolumeSpecName, + volumeToMount.VolumeGidValue) + if markVolMountedErr != nil { + // On failure, return error. Caller will log and retry. + return volumeToMount.GenerateErrorDetailed("MapVolume.MarkVolumeAsMounted failed", markVolMountedErr) + } + + return nil + }, blockVolumePlugin.GetPluginName(), nil +} + +// GenerateUnmapVolumeFunc marks volume as unmonuted based on following steps. +// Remove symbolic links from pod device map path dir and global map path dir. +// Once those cleanups are done, remove pod device map path dir. +// If all steps are completed, the volume is marked as unmounted. +func (og *operationGenerator) GenerateUnmapVolumeFunc( + volumeToUnmount MountedVolume, + actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { + + // Get block volume unmapper plugin + var blockVolumeUnmapper volume.BlockVolumeUnmapper + blockVolumePlugin, err := + og.volumePluginMgr.FindMapperPluginByName(volumeToUnmount.PluginName) + if err != nil { + return nil, "", volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed", err) + } + if blockVolumePlugin == nil { + return nil, "", volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil) + } + blockVolumeUnmapper, newUnmapperErr := blockVolumePlugin.NewBlockVolumeUnmapper( + volumeToUnmount.InnerVolumeSpecName, volumeToUnmount.PodUID) + if newUnmapperErr != nil { + return nil, blockVolumePlugin.GetPluginName(), volumeToUnmount.GenerateErrorDetailed("UnmapVolume.NewUnmapper failed", newUnmapperErr) + } + + return func() error { + // Try to unmap volumeName symlink under pod device map path dir + // pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName} + podDeviceUnmapPath, volName := blockVolumeUnmapper.GetPodDeviceMapPath() + unmapDeviceErr := og.blkUtil.UnmapDevice(podDeviceUnmapPath, volName) + if unmapDeviceErr != nil { + // On failure, return error. Caller will log and retry. + return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.UnmapDevice on pod device map path failed", unmapDeviceErr) + } + // Try to unmap podUID symlink under global map path dir + // plugins/kubernetes.io/{PluginName}/volumeDevices/{volumePluginDependentPath}/{podUID} + globalUnmapPath, err := + blockVolumeUnmapper.GetGlobalMapPath(volumeToUnmount.VolumeSpec) + if err != nil { + // On failure, return error. Caller will log and retry. + return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.GetGlobalUnmapPath failed", err) + } + unmapDeviceErr = og.blkUtil.UnmapDevice(globalUnmapPath, string(volumeToUnmount.PodUID)) + if unmapDeviceErr != nil { + // On failure, return error. Caller will log and retry. + return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.UnmapDevice on global map path failed", unmapDeviceErr) + } + + glog.Infof( + "UnmapVolume succeeded for volume %q (OuterVolumeSpecName: %q) pod %q (UID: %q). InnerVolumeSpecName %q. PluginName %q, VolumeGidValue %q", + volumeToUnmount.VolumeName, + volumeToUnmount.OuterVolumeSpecName, + volumeToUnmount.PodName, + volumeToUnmount.PodUID, + volumeToUnmount.InnerVolumeSpecName, + volumeToUnmount.PluginName, + volumeToUnmount.VolumeGidValue) + + // Update actual state of world + markVolUnmountedErr := actualStateOfWorld.MarkVolumeAsUnmounted( + volumeToUnmount.PodName, volumeToUnmount.VolumeName) + if markVolUnmountedErr != nil { + // On failure, just log and exit + glog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmapVolume.MarkVolumeAsUnmounted failed", markVolUnmountedErr).Error()) + } + + return nil + }, blockVolumePlugin.GetPluginName(), nil +} + +// GenerateUnmapDeviceFunc marks device as unmounted based on following steps. +// Check under globalMapPath dir if there isn't pod's symbolic links in it. +// If symbolick link isn't there, the device isn't referenced from Pods. +// Call plugin TearDownDevice to clean-up device connection, stored data under +// globalMapPath, these operations depend on plugin implementation. +// Once TearDownDevice is completed, remove globalMapPath dir. +// After globalMapPath is removed, fd lock by loopback for the device can +// be released safely because no one can consume the device at this point. +// At last, device open status will be checked just in case. +// If all steps are completed, the device is marked as unmounted. +func (og *operationGenerator) GenerateUnmapDeviceFunc( + deviceToDetach AttachedVolume, + actualStateOfWorld ActualStateOfWorldMounterUpdater, + mounter mount.Interface) (func() error, string, error) { + + // Get block volume mapper plugin + var blockVolumeMapper volume.BlockVolumeMapper + blockVolumePlugin, err := + og.volumePluginMgr.FindMapperPluginBySpec(deviceToDetach.VolumeSpec) + if err != nil { + return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginBySpec failed", err) + } + if blockVolumePlugin == nil { + return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil) + } + blockVolumeMapper, newMapperErr := blockVolumePlugin.NewBlockVolumeMapper( + deviceToDetach.VolumeSpec, + nil, /* Pod */ + volume.VolumeOptions{}) + if newMapperErr != nil { + return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.NewBlockVolumeMapper initialization failed", newMapperErr) + } + + blockVolumeUnmapper, newUnmapperErr := blockVolumePlugin.NewBlockVolumeUnmapper( + string(deviceToDetach.VolumeName), + "" /* podUID */) + if newUnmapperErr != nil { + return nil, blockVolumePlugin.GetPluginName(), deviceToDetach.GenerateErrorDetailed("UnmapDevice.NewUnmapper failed", newUnmapperErr) + } + + return func() error { + // Search under globalMapPath dir if all symbolic links from pods have been removed already. + // If symbolick links are there, pods may still refer the volume. + globalMapPath, err := + blockVolumeMapper.GetGlobalMapPath(deviceToDetach.VolumeSpec) + if err != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateErrorDetailed("UnmapDevice.GetGlobalMapPath failed", err) + } + refs, err := og.blkUtil.GetDeviceSymlinkRefs(deviceToDetach.DevicePath, globalMapPath) + if err != nil { + return deviceToDetach.GenerateErrorDetailed("UnmapDevice.GetDeviceSymlinkRefs check failed", err) + } + if len(refs) > 0 { + err = fmt.Errorf("The device %q is still referenced from other Pods %v", globalMapPath, refs) + return deviceToDetach.GenerateErrorDetailed("UnmapDevice failed", err) + } + + // Execute tear down device + unmapErr := blockVolumeUnmapper.TearDownDevice(globalMapPath, deviceToDetach.DevicePath) + if unmapErr != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateErrorDetailed("UnmapDevice.TearDownDevice failed", unmapErr) + } + + // Plugin finished TearDownDevice(). Now globalMapPath dir and plugin's stored data + // on the dir are unnecessary, clean up it. + removeMapPathErr := og.blkUtil.RemoveMapPath(globalMapPath) + if removeMapPathErr != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateErrorDetailed("UnmapDevice failed", removeMapPathErr) + } + + // The block volume is not referenced from Pods. Release file descriptor lock. + glog.V(5).Infof("UnmapDevice: deviceToDetach.DevicePath: %v", deviceToDetach.DevicePath) + loopPath, err := og.blkUtil.GetLoopDevice(deviceToDetach.DevicePath) + if err != nil { + glog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath))) + } else { + err = og.blkUtil.RemoveLoopDevice(loopPath) + if err != nil { + return deviceToDetach.GenerateErrorDetailed("UnmapDevice.AttachFileDevice failed", err) + } + } + + // Before logging that UnmapDevice succeeded and moving on, + // use mounter.PathIsDevice to check if the path is a device, + // if so use mounter.DeviceOpened to check if the device is in use anywhere + // else on the system. Retry if it returns true. + deviceOpened, deviceOpenedErr := isDeviceOpened(deviceToDetach, mounter) + if deviceOpenedErr != nil { + return deviceOpenedErr + } + // The device is still in use elsewhere. Caller will log and retry. + if deviceOpened { + return deviceToDetach.GenerateErrorDetailed( + "UnmapDevice failed", + fmt.Errorf("the device is in use when it was no longer expected to be in use")) + } + + glog.Infof(deviceToDetach.GenerateMsgDetailed("UnmapDevice succeeded", "")) + + // Update actual state of world + markDeviceUnmountedErr := actualStateOfWorld.MarkDeviceAsUnmounted( + deviceToDetach.VolumeName) + if markDeviceUnmountedErr != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateErrorDetailed("MarkDeviceAsUnmounted failed", markDeviceUnmountedErr) + } + + return nil + }, blockVolumePlugin.GetPluginName(), nil +} + func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc( volumeToMount VolumeToMount, nodeName types.NodeName, @@ -833,3 +1170,23 @@ func checkNodeAffinity(og *operationGenerator, volumeToMount VolumeToMount, plug } return nil } + +// isDeviceOpened checks the device status if the device is in use anywhere else on the system +func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (bool, error) { + isDevicePath, devicePathErr := mounter.PathIsDevice(deviceToDetach.DevicePath) + var deviceOpened bool + var deviceOpenedErr error + if !isDevicePath && devicePathErr == nil || + (devicePathErr != nil && strings.Contains(devicePathErr.Error(), "does not exist")) { + // not a device path or path doesn't exist + //TODO: refer to #36092 + glog.V(3).Infof("The path isn't device path or doesn't exist. Skip checking device path: %s", deviceToDetach.DevicePath) + deviceOpened = false + } else { + deviceOpened, deviceOpenedErr = mounter.DeviceOpened(deviceToDetach.DevicePath) + if deviceOpenedErr != nil { + return false, deviceToDetach.GenerateErrorDetailed("DeviceOpened failed", deviceOpenedErr) + } + } + return deviceOpened, nil +} diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index 2bd4fc89c28..7f7e2d9a8c8 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -21,7 +21,7 @@ import ( "io/ioutil" "os" "path" - + "path/filepath" "strings" "github.com/golang/glog" @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -38,7 +39,14 @@ import ( "k8s.io/kubernetes/pkg/util/mount" ) -const readyFileName = "ready" +const ( + readyFileName = "ready" + losetupPath = "losetup" + + ErrDeviceNotFound = "device not found" + ErrDeviceNotSupported = "device not supported" + ErrNotAvailable = "not available" +) // IsReady checks for the existence of a regular file // called 'ready' in the given directory and returns @@ -270,3 +278,201 @@ func stringToSet(str, delimiter string) (sets.String, error) { } return zonesSet, nil } + +// BlockVolumePathHandler defines a set of operations for handling block volume-related operations +type BlockVolumePathHandler interface { + // MapDevice creates a symbolic link to block device under specified map path + MapDevice(devicePath string, mapPath string, linkName string) error + // UnmapDevice removes a symbolic link to block device under specified map path + UnmapDevice(mapPath string, linkName string) error + // RemovePath removes a file or directory on specified map path + RemoveMapPath(mapPath string) error + // IsSymlinkExist retruns true if specified symbolic link exists + IsSymlinkExist(mapPath string) (bool, error) + // GetDeviceSymlinkRefs searches symbolic links under global map path + GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) + // FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath + // corresponding to map path symlink, and then return global map path with pod uuid. + FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) + // AttachFileDevice takes a path to a regular file and makes it available as an + // attached block device. + AttachFileDevice(path string) (string, error) + // GetLoopDevice returns the full path to the loop device associated with the given path. + GetLoopDevice(path string) (string, error) + // RemoveLoopDevice removes specified loopback device + RemoveLoopDevice(device string) error +} + +// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler. +func NewBlockVolumePathHandler() BlockVolumePathHandler { + var volumePathHandler VolumePathHandler + return volumePathHandler +} + +// VolumePathHandler is path related operation handlers for block volume +type VolumePathHandler struct { +} + +// MapDevice creates a symbolic link to block device under specified map path +func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string) error { + // Example of global map path: + // globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid} + // linkName: {podUid} + // + // Example of pod device map path: + // podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} + // linkName: {volumeName} + if len(devicePath) == 0 { + return fmt.Errorf("Failed to map device to map path. devicePath is empty") + } + if len(mapPath) == 0 { + return fmt.Errorf("Failed to map device to map path. mapPath is empty") + } + if !filepath.IsAbs(mapPath) { + return fmt.Errorf("The map path should be absolute: map path: %s", mapPath) + } + glog.V(5).Infof("MapDevice: devicePath %s", devicePath) + glog.V(5).Infof("MapDevice: mapPath %s", mapPath) + glog.V(5).Infof("MapDevice: linkName %s", linkName) + + // Check and create mapPath + _, err := os.Stat(mapPath) + if err != nil && !os.IsNotExist(err) { + glog.Errorf("cannot validate map path: %s", mapPath) + return err + } + if err = os.MkdirAll(mapPath, 0750); err != nil { + return fmt.Errorf("Failed to mkdir %s, error %v", mapPath, err) + } + // Remove old symbolic link(or file) then create new one. + // This should be done because current symbolic link is + // stale accross node reboot. + linkPath := path.Join(mapPath, string(linkName)) + if err = os.Remove(linkPath); err != nil && !os.IsNotExist(err) { + return err + } + err = os.Symlink(devicePath, linkPath) + return err +} + +// UnmapDevice removes a symbolic link associated to block device under specified map path +func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error { + if len(mapPath) == 0 { + return fmt.Errorf("Failed to unmap device from map path. mapPath is empty") + } + glog.V(5).Infof("UnmapDevice: mapPath %s", mapPath) + glog.V(5).Infof("UnmapDevice: linkName %s", linkName) + + // Check symbolic link exists + linkPath := path.Join(mapPath, string(linkName)) + if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil { + return checkErr + } else if !islinkExist { + glog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath) + return nil + } + err := os.Remove(linkPath) + return err +} + +// RemoveMapPath removes a file or directory on specified map path +func (v VolumePathHandler) RemoveMapPath(mapPath string) error { + if len(mapPath) == 0 { + return fmt.Errorf("Failed to remove map path. mapPath is empty") + } + glog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath) + err := os.RemoveAll(mapPath) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// IsSymlinkExist returns true if specified file exists and the type is symbolik link. +// If file doesn't exist, or file exists but not symbolick link, return false with no error. +// On other cases, return false with error from Lstat(). +func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) { + fi, err := os.Lstat(mapPath) + if err == nil { + // If file exits and it's symbolick link, return true and no error + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + return true, nil + } + // If file exits but it's not symbolick link, return fale and no error + return false, nil + } + // If file doesn't exist, return false and no error + if os.IsNotExist(err) { + return false, nil + } + // Return error from Lstat() + return false, err +} + +// GetDeviceSymlinkRefs searches symbolic links under global map path +func (v VolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) { + var refs []string + files, err := ioutil.ReadDir(mapPath) + if err != nil { + return nil, fmt.Errorf("Directory cannot read %v", err) + } + for _, file := range files { + if file.Mode()&os.ModeSymlink != os.ModeSymlink { + continue + } + filename := file.Name() + filepath, err := os.Readlink(path.Join(mapPath, filename)) + if err != nil { + return nil, fmt.Errorf("Symbolic link cannot be retrieved %v", err) + } + glog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", filepath, devPath) + if filepath == devPath { + refs = append(refs, path.Join(mapPath, filename)) + } + } + glog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs) + return refs, nil +} + +// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath +// corresponding to map path symlink, and then return global map path with pod uuid. +// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX +// globalMapPath/{pod uuid}: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX +func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) { + var globalMapPathUUID string + // Find symbolic link named pod uuid under plugin dir + err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if (fi.Mode()&os.ModeSymlink == os.ModeSymlink) && (fi.Name() == string(podUID)) { + glog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath) + if res, err := compareSymlinks(path, mapPath); err == nil && res { + globalMapPathUUID = path + } + } + return nil + }) + if err != nil { + return "", err + } + glog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID) + // Return path contains global map path + {pod uuid} + return globalMapPathUUID, nil +} + +func compareSymlinks(global, pod string) (bool, error) { + devGlobal, err := os.Readlink(global) + if err != nil { + return false, err + } + devPod, err := os.Readlink(pod) + if err != nil { + return false, err + } + glog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod) + if devGlobal == devPod { + return true, nil + } + return false, nil +} diff --git a/pkg/volume/util/util_linux.go b/pkg/volume/util/util_linux.go new file mode 100644 index 00000000000..755a10f012a --- /dev/null +++ b/pkg/volume/util/util_linux.go @@ -0,0 +1,106 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/golang/glog" +) + +// AttachFileDevice takes a path to a regular file and makes it available as an +// attached block device. +func (v VolumePathHandler) AttachFileDevice(path string) (string, error) { + blockDevicePath, err := v.GetLoopDevice(path) + if err != nil && err.Error() != ErrDeviceNotFound { + return "", err + } + + // If no existing loop device for the path, create one + if blockDevicePath == "" { + glog.V(4).Infof("Creating device for path: %s", path) + blockDevicePath, err = makeLoopDevice(path) + if err != nil { + return "", err + } + } + return blockDevicePath, nil +} + +// GetLoopDevice returns the full path to the loop device associated with the given path. +func (v VolumePathHandler) GetLoopDevice(path string) (string, error) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return "", errors.New(ErrNotAvailable) + } + if err != nil { + return "", fmt.Errorf("not attachable: %v", err) + } + + args := []string{"-j", path} + cmd := exec.Command(losetupPath, args...) + out, err := cmd.CombinedOutput() + if err != nil { + glog.V(2).Infof("Failed device discover command for path %s: %v", path, err) + return "", err + } + return parseLosetupOutputForDevice(out) +} + +func makeLoopDevice(path string) (string, error) { + args := []string{"-f", "--show", path} + cmd := exec.Command(losetupPath, args...) + out, err := cmd.CombinedOutput() + if err != nil { + glog.V(2).Infof("Failed device create command for path %s: %v", path, err) + return "", err + } + return parseLosetupOutputForDevice(out) +} + +// RemoveLoopDevice removes specified loopback device +func (v VolumePathHandler) RemoveLoopDevice(device string) error { + args := []string{"-d", device} + cmd := exec.Command(losetupPath, args...) + out, err := cmd.CombinedOutput() + if err != nil { + if !strings.Contains(string(out), "No such device or address") { + return err + } + } + return nil +} + +func parseLosetupOutputForDevice(output []byte) (string, error) { + if len(output) == 0 { + return "", errors.New(ErrDeviceNotFound) + } + + // losetup returns device in the format: + // /dev/loop1: [0073]:148662 (/dev/sda) + device := strings.TrimSpace(strings.SplitN(string(output), ":", 2)[0]) + if len(device) == 0 { + return "", errors.New(ErrDeviceNotFound) + } + return device, nil +} diff --git a/pkg/volume/util/util_unsupported.go b/pkg/volume/util/util_unsupported.go new file mode 100644 index 00000000000..930e4f663dd --- /dev/null +++ b/pkg/volume/util/util_unsupported.go @@ -0,0 +1,39 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" +) + +// AttachFileDevice takes a path to a regular file and makes it available as an +// attached block device. +func (v VolumePathHandler) AttachFileDevice(path string) (string, error) { + return "", fmt.Errorf("AttachFileDevice not supported for this build.") +} + +// GetLoopDevice returns the full path to the loop device associated with the given path. +func (v VolumePathHandler) GetLoopDevice(path string) (string, error) { + return "", fmt.Errorf("GetLoopDevice not supported for this build.") +} + +// RemoveLoopDevice removes specified loopback device +func (v VolumePathHandler) RemoveLoopDevice(device string) error { + return fmt.Errorf("RemoveLoopDevice not supported for this build.") +} diff --git a/pkg/volume/util/volumehelper/volumehelper.go b/pkg/volume/util/volumehelper/volumehelper.go index b0734601d0a..74b14be5de8 100644 --- a/pkg/volume/util/volumehelper/volumehelper.go +++ b/pkg/volume/util/volumehelper/volumehelper.go @@ -136,3 +136,24 @@ func NewSafeFormatAndMountFromHost(pluginName string, host volume.VolumeHost) *m exec := host.GetExec(pluginName) return &mount.SafeFormatAndMount{Interface: mounter, Exec: exec} } + +// GetVolumeMode retrieves VolumeMode from pv. +// If the volume doesn't have PersistentVolume, it's an inline volume, +// should return volumeMode as filesystem to keep existing behavior. +func GetVolumeMode(volumeSpec *volume.Spec) (v1.PersistentVolumeMode, error) { + if volumeSpec == nil || volumeSpec.PersistentVolume == nil { + return v1.PersistentVolumeFilesystem, nil + } + if volumeSpec.PersistentVolume.Spec.VolumeMode != nil { + return *volumeSpec.PersistentVolume.Spec.VolumeMode, nil + } + return "", fmt.Errorf("cannot get volumeMode for volume: %v", volumeSpec.Name()) +} + +// GetPersistentVolumeClaimVolumeMode retrieves VolumeMode from pvc. +func GetPersistentVolumeClaimVolumeMode(claim *v1.PersistentVolumeClaim) (v1.PersistentVolumeMode, error) { + if claim.Spec.VolumeMode != nil { + return *claim.Spec.VolumeMode, nil + } + return "", fmt.Errorf("cannot get volumeMode from pvc: %v", claim.Name) +} diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 23e395299e8..471963556a8 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -37,6 +37,19 @@ type Volume interface { MetricsProvider } +// BlockVolume interface provides methods to generate global map path +// and pod device map path. +type BlockVolume interface { + // GetGlobalMapPath returns a global map path which contains + // symbolic links associated to a block device. + // ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} + GetGlobalMapPath(spec *Spec) (string, error) + // GetPodDeviceMapPath returns a pod device map path + // and name of a symbolic link associated to a block device. + // ex. pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} + GetPodDeviceMapPath() (string, string) +} + // MetricsProvider exposes metrics (e.g. used,available space) related to a // Volume. type MetricsProvider interface { @@ -132,6 +145,34 @@ type Unmounter interface { TearDownAt(dir string) error } +// BlockVolumeMapper interface provides methods to set up/map the volume. +type BlockVolumeMapper interface { + BlockVolume + // SetUpDevice prepares the volume to a self-determined directory path, + // which may or may not exist yet and returns combination of physical + // device path of a block volume and error. + // If the plugin is non-attachable, it should prepare the device + // in /dev/ (or where appropriate) and return unique device path. + // Unique device path across kubelet node reboot is required to avoid + // unexpected block volume destruction. + // If the plugin is attachable, it should not do anything here, + // just return empty string for device path. + // Instead, attachable plugin have to return unique device path + // at attacher.Attach() and attacher.WaitForAttach(). + // This may be called more than once, so implementations must be idempotent. + SetUpDevice() (string, error) +} + +// BlockVolumeUnmapper interface provides methods to cleanup/unmap the volumes. +type BlockVolumeUnmapper interface { + BlockVolume + // TearDownDevice removes traces of the SetUpDevice procedure under + // a self-determined directory. + // If the plugin is non-attachable, this method detaches the volume + // from a node. + TearDownDevice(mapPath string, devicePath string) error +} + // Provisioner is an interface that creates templates for PersistentVolumes // and can create the volume as a new resource in the infrastructure provider. type Provisioner interface {