From 64e8396e300398dd0a2828c89a3e44fe90889e8c Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 10 May 2021 10:49:31 +0200 Subject: [PATCH] Retry detaching FibreChannel volume few times When UnmountDevice() of a FibreChannel volume fails after unmounting the device and before the device is fully cleaned up, subsequent UnmountDevice() retry won't find the device mounted and return without retrying the device cleanup. Therefore implement its own retry inside UnmountDevice() to make sure that the volume devices are either fully cleaned or the error is serius enough that even 1 minute of trying does not help. --- pkg/volume/fc/attacher.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/volume/fc/attacher.go b/pkg/volume/fc/attacher.go index c94c95d47a6..7b6495106a3 100644 --- a/pkg/volume/fc/attacher.go +++ b/pkg/volume/fc/attacher.go @@ -23,13 +23,13 @@ import ( "strings" "time" - "k8s.io/klog/v2" - "k8s.io/mount-utils" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/mount-utils" ) type fcAttacher struct { @@ -172,12 +172,28 @@ func (detacher *fcDetacher) UnmountDevice(deviceMountPath string) error { if devName == "" { return nil } + unMounter := volumeSpecToUnmounter(detacher.mounter, detacher.host) - err = detacher.manager.DetachDisk(*unMounter, devName) + // The device is unmounted now. If UnmountDevice was retried, GetDeviceNameFromMount + // won't find any mount and won't return DetachDisk below. + // Therefore implement our own retry mechanism here. + // E.g. DetachDisk sometimes fails to flush a multipath device with "device is busy" when it was + // just unmounted. + // 2 minutes should be enough within 6 minute force detach timeout. + var detachError error + err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { + detachError = detacher.manager.DetachDisk(*unMounter, devName) + if detachError != nil { + klog.V(4).Infof("fc: failed to detach disk %s (%s): %v", devName, deviceMountPath, detachError) + return false, nil + } + return true, nil + }) if err != nil { - return fmt.Errorf("fc: failed to detach disk: %s\nError: %v", devName, err) + return fmt.Errorf("fc: failed to detach disk: %s: %v", devName, detachError) } - klog.V(4).Infof("fc: successfully detached disk: %s", devName) + + klog.V(2).Infof("fc: successfully detached disk: %s", devName) return nil }