github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #37255 from jingxu97/Nov/nfshung

Browse Source 
Automatic merge from submit-queue

remove checking mount point in cleanupOrphanedPodDirs

To avoid nfs hung problem, remove the mountpoint checking code in
cleanupOrphanedPodDirs(). This removal should still be safe because it checks whether there are still directories under pod's volume and if so, do not delete the pod directory.

Note: After removing the mountpoint check code in cleanupOrphanedPodDirs(), the directories might not be cleaned up in such situation.
1. delete pod, kubelet reconciler tries to unmount the volume directory successfully
2. before reconciler tries to delete the volume directory, kubelet gets retarted
3. since under pod directory, there are still volume directors exist (but not mounted), cleanupOrphanedPodDIrs() will not clean them up.

Will work on a follow up PR to solve above issue.
This commit is contained in:
Kubernetes Submit Queue 2016-11-30 21:11:13 -08:00 committed by GitHub
commit 2ed490e15b
2 changed files with 4 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/kubelet/kubelet.go
View File

@ -1851,8 +1851,8 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle
}
case <-housekeepingCh:
if !kl.sourcesReady.AllReady() {
// If the sources aren't ready, skip housekeeping, as we may
// accidentally delete pods from unready sources.
// If the sources aren't ready or volume manager has not yet synced the states,
// skip housekeeping, as we may accidentally delete pods from unready sources.
glog.V(4).Infof("SyncLoop (housekeeping, skipped): sources aren't ready yet.")
} else {
glog.V(4).Infof("SyncLoop (housekeeping)")

17
pkg/kubelet/kubelet_volumes.go
View File

@ -25,7 +25,6 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/types"
utilerrors "k8s.io/kubernetes/pkg/util/errors"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/pkg/volume"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
@ -105,24 +104,12 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(
glog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
continue
}
// Check whether volume is still mounted on disk. If so, do not delete directory
// If there are still volume directories, do not delete directory
volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
if err != nil {
if err != nil || len(volumePaths) > 0 {
glog.Errorf("Orphaned pod %q found, but error %v occured during reading volume dir from disk", uid, err)
continue
} else if len(volumePaths) > 0 {
for _, path := range volumePaths {
notMount, err := mount.IsNotMountPoint(path)
if err == nil && notMount {
glog.V(2).Infof("Volume path %q is no longer mounted, remove it", path)
os.Remove(path)
} else {
glog.Errorf("Orphaned pod %q found, but it might still mounted with error %v", uid, err)
}
}
continue
}
glog.V(3).Infof("Orphaned pod %q found, removing", uid)
if err := os.RemoveAll(kl.getPodDir(uid)); err != nil {
glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)