Merge pull request #70695 from mborsz/DeleteResourceAndWaitForGC

Few improvements to DeleteResourceAndWaitForGC
This commit is contained in:
k8s-ci-robot 2018-11-07 02:47:37 -08:00 committed by GitHub
commit 465d578d93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -3161,7 +3161,10 @@ func DeleteResourceAndWaitForGC(c clientset.Interface, kind schema.GroupKind, ns
terminatePodTime := time.Since(startTime) - deleteTime
Logf("Terminating %v %s pods took: %v", kind, name, terminatePodTime)
err = waitForPodsGone(ps, interval, 10*time.Minute)
// In gce, at any point, small percentage of nodes can disappear for
// ~10 minutes due to hostError. 20 minutes should be long enough to
// restart VM in that case and delete the pod.
err = waitForPodsGone(ps, interval, 20*time.Minute)
if err != nil {
return fmt.Errorf("error while waiting for pods gone %s: %v", name, err)
}
@ -3191,21 +3194,30 @@ func waitForPodsInactive(ps *testutils.PodStore, interval, timeout time.Duration
if err == wait.ErrWaitTimeout {
for _, pod := range activePods {
Logf("Pod %q running on %q is still active", pod.ObjectMeta.Name, pod.Spec.NodeName)
Logf("ERROR: Pod %q running on %q is still active", pod.Name, pod.Spec.NodeName)
}
return fmt.Errorf("there are %d active pods. E.g. %q on node %q", len(activePods), activePods[0].ObjectMeta.Name, activePods[0].Spec.NodeName)
return fmt.Errorf("there are %d active pods. E.g. %q on node %q", len(activePods), activePods[0].Name, activePods[0].Spec.NodeName)
}
return err
}
// waitForPodsGone waits until there are no pods left in the PodStore.
func waitForPodsGone(ps *testutils.PodStore, interval, timeout time.Duration) error {
return wait.PollImmediate(interval, timeout, func() (bool, error) {
if pods := ps.List(); len(pods) == 0 {
var pods []*v1.Pod
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
if pods = ps.List(); len(pods) == 0 {
return true, nil
}
return false, nil
})
if err == wait.ErrWaitTimeout {
for _, pod := range pods {
Logf("ERROR: Pod %q still exists. Node: %q", pod.Name, pod.Spec.NodeName)
}
return fmt.Errorf("there are %d pods left. E.g. %q on node %q", len(pods), pods[0].Name, pods[0].Spec.NodeName)
}
return err
}
func WaitForPodsReady(c clientset.Interface, ns, name string, minReadySeconds int) error {