diff --git a/pkg/volume/cinder/attacher.go b/pkg/volume/cinder/attacher.go index 1bd25712425..95a28c82496 100644 --- a/pkg/volume/cinder/attacher.go +++ b/pkg/volume/cinder/attacher.go @@ -44,7 +44,9 @@ var _ volume.Attacher = &cinderDiskAttacher{} var _ volume.AttachableVolumePlugin = &cinderPlugin{} const ( - checkSleepDuration = 5 * time.Second + probeVolumeInitDealy = 1 * time.Second + probeVolumeFactor = 2.0 + probeVolumeSteps = 10 operationFinishInitDealy = 1 * time.Second operationFinishFactor = 1.1 operationFinishSteps = 10 @@ -221,6 +223,38 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod return volumesAttachedCheck, nil } +func (attacher *cinderDiskAttacher) waitProbeVolume(devicePath, volumeID string) (string, error) { + backoff := wait.Backoff{ + Duration: probeVolumeInitDealy, + Factor: probeVolumeFactor, + Steps: probeVolumeSteps, + } + + err := wait.ExponentialBackoff(backoff, func() (string, error) { + glog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID) + probeAttachedVolume() + if !attacher.cinderProvider.ShouldTrustDevicePath() { + // Using the Cinder volume ID, find the real device path (See Issue #33128) + devicePath = attacher.cinderProvider.GetDevicePath(volumeID) + } + exists, err := volumeutil.PathExists(devicePath) + if exists && err == nil { + glog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath) + return devicePath, nil + } + else { + // Log an error, and continue checking periodically + glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err) + } + }) + + if err == wait.ErrWaitTimeout { + err = fmt.Errorf("Volume %q failed to be probed within the alloted time", volumeID) + } + + return "", err +} + func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { // NOTE: devicePath is is path as reported by Cinder, which may be incorrect and should not be used. See Issue #33128 volumeSource, _, err := getVolumeSource(spec) @@ -234,32 +268,8 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath return "", fmt.Errorf("WaitForAttach failed for Cinder disk %q: devicePath is empty.", volumeID) } - ticker := time.NewTicker(checkSleepDuration) - defer ticker.Stop() - timer := time.NewTimer(timeout) - defer timer.Stop() - - for { - select { - case <-ticker.C: - glog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID) - probeAttachedVolume() - if !attacher.cinderProvider.ShouldTrustDevicePath() { - // Using the Cinder volume ID, find the real device path (See Issue #33128) - devicePath = attacher.cinderProvider.GetDevicePath(volumeID) - } - exists, err := volumeutil.PathExists(devicePath) - if exists && err == nil { - glog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath) - return devicePath, nil - } else { - // Log an error, and continue checking periodically - glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err) - } - case <-timer.C: - return "", fmt.Errorf("Could not find attached Cinder disk %q. Timeout waiting for mount paths to be created.", volumeID) - } - } + // Using exponential backoff instead of linear + return attacher.waitProbeVolume(devicePath, volumeID) } func (attacher *cinderDiskAttacher) GetDeviceMountPath( diff --git a/pkg/volume/cinder/cinder_util.go b/pkg/volume/cinder/cinder_util.go index df258a06293..d994dff0b1a 100644 --- a/pkg/volume/cinder/cinder_util.go +++ b/pkg/volume/cinder/cinder_util.go @@ -228,12 +228,11 @@ func probeAttachedVolume() error { // udevadm settle waits for udevd to process the device creation // events for all hardware devices, thus ensuring that any device // nodes have been created successfully before proceeding. - argsSettle := []string{"settle", "--timeout=1"} + argsSettle := []string{"settle"} cmdSettle := executor.Command("udevadm", argsSettle...) _, errSettle := cmdSettle.CombinedOutput() if errSettle != nil { glog.Errorf("error running udevadm settle %v\n", errSettle) - return errSettle } args := []string{"trigger"}