diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index 6d237e7f052..0d18e1832fb 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -42,6 +42,7 @@ go_library( "//vendor/google.golang.org/grpc/codes:go_default_library", "//vendor/google.golang.org/grpc/status:go_default_library", "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/strings:go_default_library", ], ) @@ -86,6 +87,7 @@ go_test( "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/csi/csi_attacher.go b/pkg/volume/csi/csi_attacher.go index e3ba528a14d..fc5c3a28bb8 100644 --- a/pkg/volume/csi/csi_attacher.go +++ b/pkg/volume/csi/csi_attacher.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -228,13 +228,19 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo return errors.New(log("attacher.MountDevice failed, deviceMountPath is empty")) } + corruptedDir := false mounted, err := isDirMounted(c.plugin, deviceMountPath) if err != nil { klog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath)) - return err + if isCorruptedDir(deviceMountPath) { + corruptedDir = true // leave to CSI driver to handle corrupted mount + klog.Warning(log("attacher.MountDevice detected corrupted mount for dir [%s]", deviceMountPath)) + } else { + return err + } } - if mounted { + if mounted && !corruptedDir { klog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath)) return nil } @@ -287,7 +293,7 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo // Store volume metadata for UnmountDevice. Keep it around even if the // driver does not support NodeStage, UnmountDevice still needs it. - if err = os.MkdirAll(deviceMountPath, 0750); err != nil { + if err = os.MkdirAll(deviceMountPath, 0750); err != nil && !corruptedDir { return errors.New(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err)) } klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath)) diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index c9d831bdf57..b807326cf72 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" + "k8s.io/utils/mount" utilstrings "k8s.io/utils/strings" ) @@ -105,12 +106,18 @@ func (c *csiMountMgr) SetUp(mounterArgs volume.MounterArgs) error { func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir)) + corruptedDir := false mounted, err := isDirMounted(c.plugin, dir) if err != nil { - return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err)) + if isCorruptedDir(dir) { + corruptedDir = true // leave to CSI driver to handle corrupted mount + klog.Warning(log("mounter.SetUpAt detected corrupted mount for dir [%s]", dir)) + } else { + return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err)) + } } - if mounted { + if mounted && !corruptedDir { klog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir)) return nil } @@ -211,7 +218,7 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error } // create target_dir before call to NodePublish - if err := os.MkdirAll(dir, 0750); err != nil { + if err := os.MkdirAll(dir, 0750); err != nil && !corruptedDir { return errors.New(log("mounter.SetUpAt failed to create dir %#v: %v", dir, err)) } klog.V(4).Info(log("created target path successfully [%s]", dir)) @@ -417,6 +424,11 @@ func isDirMounted(plug *csiPlugin, dir string) (bool, error) { return !notMnt, nil } +func isCorruptedDir(dir string) bool { + _, pathErr := mount.PathExists(dir) + return pathErr != nil && mount.IsCorruptedMnt(pathErr) +} + // removeMountDir cleans the mount dir when dir is not mounted and removed the volume data file in dir func removeMountDir(plug *csiPlugin, mountPath string) error { klog.V(4).Info(log("removing mount path [%s]", mountPath)) diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index 70c1d492452..5a83a33b6a5 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -19,6 +19,7 @@ package csi import ( "context" "fmt" + "io/ioutil" "math/rand" "os" "path" @@ -27,6 +28,8 @@ import ( "reflect" + "github.com/stretchr/testify/assert" + api "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -825,3 +828,33 @@ func TestUnmounterTeardown(t *testing.T) { } } + +func TestIsCorruptedDir(t *testing.T) { + existingMountPath, err := ioutil.TempDir(os.TempDir(), "blobfuse-csi-mount-test") + if err != nil { + t.Fatalf("failed to create tmp dir: %v", err) + } + defer os.RemoveAll(existingMountPath) + + tests := []struct { + desc string + dir string + expectedResult bool + }{ + { + desc: "NotExist dir", + dir: "/tmp/NotExist", + expectedResult: false, + }, + { + desc: "Existing dir", + dir: existingMountPath, + expectedResult: false, + }, + } + + for i, test := range tests { + isCorruptedDir := isCorruptedDir(test.dir) + assert.Equal(t, test.expectedResult, isCorruptedDir, "TestCase[%d]: %s", i, test.desc) + } +}