github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #55944 from edisonxiang/probeimprovecinder

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 55925, 55999, 55944, 55992, 56196). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

probeAttachedVolume improvement in Cinder

What this PR does / why we need it:

During OpenStack Cinder volume attachment, kubelet creates too many udev "change" events.
If volume attachement takes long, then udev event queue gets full,therefore
udev does not answer requests from docker, and makes Kubernetes node unhealthy.

This patch will extend the udevadm trigger period,
and use the udevadm settle to ensure that any device nodes
have been created successfully before proceeding.

Fixes #55007

Release note:

```release-note
None
```

/cc @dims @kabakaev
This commit is contained in:
Kubernetes Submit Queue 2017-12-14 04:32:16 -08:00 committed by GitHub
commit df053ae34a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pkg/volume/cinder/attacher.go
View File

@ -44,14 +44,15 @@ var _ volume.Attacher = &cinderDiskAttacher{}
var _ volume.AttachableVolumePlugin = &cinderPlugin{}
const (
checkSleepDuration = 1 * time.Second
operationFinishInitDealy = 1 * time.Second
probeVolumeInitDelay = 1 * time.Second
probeVolumeFactor = 2.0
operationFinishInitDelay = 1 * time.Second
operationFinishFactor = 1.1
operationFinishSteps = 10
diskAttachInitDealy = 1 * time.Second
diskAttachInitDelay = 1 * time.Second
diskAttachFactor = 1.2
diskAttachSteps = 15
diskDetachInitDealy = 1 * time.Second
diskDetachInitDelay = 1 * time.Second
diskDetachFactor = 1.2
diskDetachSteps = 13
)
@ -74,7 +75,7 @@ func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string
func (attacher *cinderDiskAttacher) waitOperationFinished(volumeID string) error {
backoff := wait.Backoff{
Duration: operationFinishInitDealy,
Duration: operationFinishInitDelay,
Factor: operationFinishFactor,
Steps: operationFinishSteps,
}
@ -99,7 +100,7 @@ func (attacher *cinderDiskAttacher) waitOperationFinished(volumeID string) error
func (attacher *cinderDiskAttacher) waitDiskAttached(instanceID, volumeID string) error {
backoff := wait.Backoff{
Duration: diskAttachInitDealy,
Duration: diskAttachInitDelay,
Factor: diskAttachFactor,
Steps: diskAttachSteps,
}
@ -234,11 +235,12 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath
return "", fmt.Errorf("WaitForAttach failed for Cinder disk %q: devicePath is empty.", volumeID)
}
ticker := time.NewTicker(checkSleepDuration)
ticker := time.NewTicker(probeVolumeInitDelay)
defer ticker.Stop()
timer := time.NewTimer(timeout)
defer timer.Stop()
duration := probeVolumeInitDelay
for {
select {
case <-ticker.C:
@ -255,6 +257,10 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath
} else {
// Log an error, and continue checking periodically
glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err)
// Using exponential backoff instead of linear
ticker.Stop()
duration = time.Duration(float64(duration) * probeVolumeFactor)
ticker = time.NewTicker(duration)
}
case <-timer.C:
return "", fmt.Errorf("Could not find attached Cinder disk %q. Timeout waiting for mount paths to be created.", volumeID)
@ -328,7 +334,7 @@ func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error {
backoff := wait.Backoff{
Duration: operationFinishInitDealy,
Duration: operationFinishInitDelay,
Factor: operationFinishFactor,
Steps: operationFinishSteps,
}
@ -353,7 +359,7 @@ func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error
func (detacher *cinderDiskDetacher) waitDiskDetached(instanceID, volumeID string) error {
backoff := wait.Backoff{
Duration: diskDetachInitDealy,
Duration: diskDetachInitDelay,
Factor: diskDetachFactor,
Steps: diskDetachSteps,
}

11
pkg/volume/cinder/cinder_util.go
View File

@ -224,6 +224,17 @@ func probeAttachedVolume() error {
scsiHostRescan()
executor := exec.New()
// udevadm settle waits for udevd to process the device creation
// events for all hardware devices, thus ensuring that any device
// nodes have been created successfully before proceeding.
argsSettle := []string{"settle"}
cmdSettle := executor.Command("udevadm", argsSettle...)
_, errSettle := cmdSettle.CombinedOutput()
if errSettle != nil {
glog.Errorf("error running udevadm settle %v\n", errSettle)
}
args := []string{"trigger"}
cmd := executor.Command("udevadm", args...)
_, err := cmd.CombinedOutput()