github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-13 11:25:19 +00:00
Code Issues Projects Releases Wiki Activity

Fix volume states out of sync problem after kubelet restarts

Browse Source 
When kubelet restarts, all the information about the volumes will be
gone from actual/desired states. When update node status with mounted
volumes, the volume list might be empty although there are still volumes
are mounted and in turn causing master to detach those volumes since
they are not in the mounted volumes list. This fix is to make sure only
update mounted volumes list after reconciler starts sync states process.
This sync state process will scan the existing volume directories and
reconstruct actual states if they are missing.

This PR also fixes the problem during orphaned pods' directories. In
case of the pod directory is unmounted but has not yet deleted (e.g.,
interrupted with kubelet restarts), clean up routine will delete the
directory so that the pod directoriy could be cleaned up (it is safe to
delete directory since it is no longer mounted)

The third issue this PR fixes is that during reconstruct volume in
actual state, mounter could not be nil since it is required for creating
container.VolumeMap. If it is nil, it might cause nil pointer exception
in kubelet.

Details are in proposal PR #33203
This commit is contained in:
Jing Xu
2016-09-27 16:12:57 -07:00
parent ebd53c6f76
commit b02481708a
11 changed files with 231 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
pkg/util/mount/mount_linux.go
View File

@@ -101,9 +101,10 @@ func isBind(options []string) (bool, []string) {
// doMount runs the mount command.
func doMount(mountCmd string, source string, target string, fstype string, options []string) error {
glog.V(5).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
glog.V(4).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
mountArgs := makeMountArgs(source, target, fstype, options)
glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs)
command := exec.Command(mountCmd, mountArgs...)
output, err := command.CombinedOutput()
if err != nil {
@@ -135,7 +136,7 @@ func makeMountArgs(source, target, fstype string, options []string) []string {
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
glog.V(5).Infof("Unmounting %s", target)
glog.V(4).Infof("Unmounting %s", target)
command := exec.Command("umount", target)
output, err := command.CombinedOutput()
if err != nil {
@@ -156,6 +157,10 @@ func (*Mounter) List() ([]MountPoint, error) {
// will return true. When in fact /tmp/b is a mount point. If this situation
// if of interest to you, don't use this function...
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
return IsNotMountPoint(file)
}
func IsNotMountPoint(file string) (bool, error) {
stat, err := os.Stat(file)
if err != nil {
return true, err
@@ -173,9 +178,10 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
}
// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// Returns true if open returns errno EBUSY, and false if errno is nil.
// Returns an error if errno is any error other than EBUSY.
// Returns with error if pathname is not a device.
// If pathname is not a device, log and return false with nil error.
// If open returns errno EBUSY, return true with nil error.
// If open returns nil, return false with nil error.
// Otherwise, return false with error
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
return exclusiveOpenFailsOnDevice(pathname)
}
@@ -187,12 +193,17 @@ func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
}
func exclusiveOpenFailsOnDevice(pathname string) (bool, error) {
if isDevice, err := pathIsDevice(pathname); !isDevice {
isDevice, err := pathIsDevice(pathname)
if err != nil {
return false, fmt.Errorf(
"PathIsDevice failed for path %q: %v",
pathname,
err)
}
if !isDevice {
glog.Errorf("Path %q is not refering to a device.", pathname)
return false, nil
}
fd, errno := syscall.Open(pathname, syscall.O_RDONLY|syscall.O_EXCL, 0)
// If the device is in use, open will return an invalid fd.
// When this happens, it is expected that Close will fail and throw an error.