mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 10:51:29 +00:00
Merge pull request #88569 from andyzhangx/csi-corrupt-mnt-fix
fix: corrupted mount point in csi driver node stage/publish
This commit is contained in:
commit
39ed64ec4c
@ -42,6 +42,7 @@ go_library(
|
|||||||
"//vendor/google.golang.org/grpc/codes:go_default_library",
|
"//vendor/google.golang.org/grpc/codes:go_default_library",
|
||||||
"//vendor/google.golang.org/grpc/status:go_default_library",
|
"//vendor/google.golang.org/grpc/status:go_default_library",
|
||||||
"//vendor/k8s.io/klog:go_default_library",
|
"//vendor/k8s.io/klog:go_default_library",
|
||||||
|
"//vendor/k8s.io/utils/mount:go_default_library",
|
||||||
"//vendor/k8s.io/utils/strings:go_default_library",
|
"//vendor/k8s.io/utils/strings:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@ -86,6 +87,7 @@ go_test(
|
|||||||
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
|
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
|
||||||
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
|
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
|
||||||
"//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library",
|
"//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library",
|
||||||
|
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||||
"//vendor/k8s.io/klog:go_default_library",
|
"//vendor/k8s.io/klog:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -28,7 +28,7 @@ import (
|
|||||||
|
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
storage "k8s.io/api/storage/v1"
|
storage "k8s.io/api/storage/v1"
|
||||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
|
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
@ -228,13 +228,19 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo
|
|||||||
return errors.New(log("attacher.MountDevice failed, deviceMountPath is empty"))
|
return errors.New(log("attacher.MountDevice failed, deviceMountPath is empty"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
corruptedDir := false
|
||||||
mounted, err := isDirMounted(c.plugin, deviceMountPath)
|
mounted, err := isDirMounted(c.plugin, deviceMountPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath))
|
klog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath))
|
||||||
return err
|
if isCorruptedDir(deviceMountPath) {
|
||||||
|
corruptedDir = true // leave to CSI driver to handle corrupted mount
|
||||||
|
klog.Warning(log("attacher.MountDevice detected corrupted mount for dir [%s]", deviceMountPath))
|
||||||
|
} else {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if mounted {
|
if mounted && !corruptedDir {
|
||||||
klog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath))
|
klog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -287,7 +293,7 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo
|
|||||||
|
|
||||||
// Store volume metadata for UnmountDevice. Keep it around even if the
|
// Store volume metadata for UnmountDevice. Keep it around even if the
|
||||||
// driver does not support NodeStage, UnmountDevice still needs it.
|
// driver does not support NodeStage, UnmountDevice still needs it.
|
||||||
if err = os.MkdirAll(deviceMountPath, 0750); err != nil {
|
if err = os.MkdirAll(deviceMountPath, 0750); err != nil && !corruptedDir {
|
||||||
return errors.New(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err))
|
return errors.New(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err))
|
||||||
}
|
}
|
||||||
klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath))
|
klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath))
|
||||||
|
@ -36,6 +36,7 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/features"
|
"k8s.io/kubernetes/pkg/features"
|
||||||
"k8s.io/kubernetes/pkg/volume"
|
"k8s.io/kubernetes/pkg/volume"
|
||||||
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
|
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
|
||||||
|
"k8s.io/utils/mount"
|
||||||
utilstrings "k8s.io/utils/strings"
|
utilstrings "k8s.io/utils/strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -105,12 +106,18 @@ func (c *csiMountMgr) SetUp(mounterArgs volume.MounterArgs) error {
|
|||||||
func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
|
func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
|
||||||
klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir))
|
klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir))
|
||||||
|
|
||||||
|
corruptedDir := false
|
||||||
mounted, err := isDirMounted(c.plugin, dir)
|
mounted, err := isDirMounted(c.plugin, dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err))
|
if isCorruptedDir(dir) {
|
||||||
|
corruptedDir = true // leave to CSI driver to handle corrupted mount
|
||||||
|
klog.Warning(log("mounter.SetUpAt detected corrupted mount for dir [%s]", dir))
|
||||||
|
} else {
|
||||||
|
return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if mounted {
|
if mounted && !corruptedDir {
|
||||||
klog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir))
|
klog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -211,7 +218,7 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
// create target_dir before call to NodePublish
|
// create target_dir before call to NodePublish
|
||||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
if err := os.MkdirAll(dir, 0750); err != nil && !corruptedDir {
|
||||||
return errors.New(log("mounter.SetUpAt failed to create dir %#v: %v", dir, err))
|
return errors.New(log("mounter.SetUpAt failed to create dir %#v: %v", dir, err))
|
||||||
}
|
}
|
||||||
klog.V(4).Info(log("created target path successfully [%s]", dir))
|
klog.V(4).Info(log("created target path successfully [%s]", dir))
|
||||||
@ -417,6 +424,11 @@ func isDirMounted(plug *csiPlugin, dir string) (bool, error) {
|
|||||||
return !notMnt, nil
|
return !notMnt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isCorruptedDir(dir string) bool {
|
||||||
|
_, pathErr := mount.PathExists(dir)
|
||||||
|
return pathErr != nil && mount.IsCorruptedMnt(pathErr)
|
||||||
|
}
|
||||||
|
|
||||||
// removeMountDir cleans the mount dir when dir is not mounted and removed the volume data file in dir
|
// removeMountDir cleans the mount dir when dir is not mounted and removed the volume data file in dir
|
||||||
func removeMountDir(plug *csiPlugin, mountPath string) error {
|
func removeMountDir(plug *csiPlugin, mountPath string) error {
|
||||||
klog.V(4).Info(log("removing mount path [%s]", mountPath))
|
klog.V(4).Info(log("removing mount path [%s]", mountPath))
|
||||||
|
@ -19,6 +19,7 @@ package csi
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
@ -27,6 +28,8 @@ import (
|
|||||||
|
|
||||||
"reflect"
|
"reflect"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
api "k8s.io/api/core/v1"
|
api "k8s.io/api/core/v1"
|
||||||
storage "k8s.io/api/storage/v1"
|
storage "k8s.io/api/storage/v1"
|
||||||
storagev1beta1 "k8s.io/api/storage/v1beta1"
|
storagev1beta1 "k8s.io/api/storage/v1beta1"
|
||||||
@ -825,3 +828,33 @@ func TestUnmounterTeardown(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsCorruptedDir(t *testing.T) {
|
||||||
|
existingMountPath, err := ioutil.TempDir(os.TempDir(), "blobfuse-csi-mount-test")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create tmp dir: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(existingMountPath)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
desc string
|
||||||
|
dir string
|
||||||
|
expectedResult bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
desc: "NotExist dir",
|
||||||
|
dir: "/tmp/NotExist",
|
||||||
|
expectedResult: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "Existing dir",
|
||||||
|
dir: existingMountPath,
|
||||||
|
expectedResult: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, test := range tests {
|
||||||
|
isCorruptedDir := isCorruptedDir(test.dir)
|
||||||
|
assert.Equal(t, test.expectedResult, isCorruptedDir, "TestCase[%d]: %s", i, test.desc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user