github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 06:54:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #42741 from kargakis/avoid-ns-skew

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 43481, 43419, 42741, 43480)

controller: work around milliseconds skew in AddAfter

AddAfter is not requeueing precisely after the provided time and may
skew for some millieseconds. This is really important because controllers
don't relist often so a missed check because of ms difference is
essentially dropping the key. For example, in [1] the test requeues a
Deployment for a progress check after 10s[2] but the Deployment is synced
9ms earlier ending up in the controller not recognizing the Deployment as
failed thus dropping it from the queue w/o any error. The drop is fixed by
forcing the controller to resync the Deployment but we are going to resync
after the full duration.

@deads2k if you don't like this I am going to handle this on a case by case basis

[1] https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133
[2] c48b2cab0f/test/e2e/deployment.go (L1122)
This commit is contained in:
Kubernetes Submit Queue 2017-03-21 19:29:27 -07:00 committed by GitHub
commit 00938eac64
4 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/controller/daemon/daemoncontroller.go
View File

@ -346,7 +346,9 @@ func (dsc *DaemonSetsController) updatePod(old, cur interface{}) {
dsc.enqueueDaemonSet(ds)
// See https://github.com/kubernetes/kubernetes/pull/38076 for more details
if changedToReady && ds.Spec.MinReadySeconds > 0 {
dsc.enqueueDaemonSetAfter(ds, time.Duration(ds.Spec.MinReadySeconds)*time.Second)
// Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
dsc.enqueueDaemonSetAfter(ds, (time.Duration(ds.Spec.MinReadySeconds)*time.Second)+time.Second)
}
return
}

4
pkg/controller/deployment/progress.go
View File

@ -233,6 +233,8 @@ func (dc *DeploymentController) requeueStuckDeployment(d *extensions.Deployment,
return time.Duration(0)
}
glog.V(4).Infof("Queueing up deployment %q for a progress check after %ds", d.Name, int(after.Seconds()))
dc.enqueueAfter(d, after)
// Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
dc.enqueueAfter(d, after+time.Second)
return after
}

4
pkg/controller/replicaset/replica_set.go
View File

@ -320,7 +320,9 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) {
// "closer" to kubelet (from the deployment to the replica set controller).
if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rs.Spec.MinReadySeconds > 0 {
glog.V(2).Infof("ReplicaSet %q will be enqueued after %ds for availability check", rs.Name, rs.Spec.MinReadySeconds)
rsc.enqueueReplicaSetAfter(rs, time.Duration(rs.Spec.MinReadySeconds)*time.Second)
// Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
rsc.enqueueReplicaSetAfter(rs, (time.Duration(rs.Spec.MinReadySeconds)*time.Second)+time.Second)
}
return
}

4
pkg/controller/replication/replication_controller.go
View File

@ -319,7 +319,9 @@ func (rm *ReplicationManager) updatePod(old, cur interface{}) {
// "closer" to kubelet (from the deployment to the ReplicationController controller).
if !v1.IsPodReady(oldPod) && v1.IsPodReady(curPod) && rc.Spec.MinReadySeconds > 0 {
glog.V(2).Infof("ReplicationController %q will be enqueued after %ds for availability check", rc.Name, rc.Spec.MinReadySeconds)
rm.enqueueControllerAfter(rc, time.Duration(rc.Spec.MinReadySeconds)*time.Second)
// Add a second to avoid milliseconds skew in AddAfter.
// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
rm.enqueueControllerAfter(rc, (time.Duration(rc.Spec.MinReadySeconds)*time.Second)+time.Second)
}
return
}