Fix error handling in gc e2e test

This commit is contained in:
jennybuckley 2018-03-14 10:27:44 -07:00
parent 209cdd9048
commit 5f518e6d4c
2 changed files with 37 additions and 36 deletions

View File

@ -55,6 +55,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",

View File

@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/storage/names" "k8s.io/apiserver/pkg/storage/names"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
@ -175,7 +176,7 @@ func verifyRemainingDeploymentsReplicaSetsPods(
} }
if len(deployments.Items) != deploymentNum { if len(deployments.Items) != deploymentNum {
ret = false ret = false
By(fmt.Sprintf("expected %d Deploymentss, got %d Deployments", deploymentNum, len(deployments.Items))) By(fmt.Sprintf("expected %d Deployments, got %d Deployments", deploymentNum, len(deployments.Items)))
} }
pods, err := clientSet.CoreV1().Pods(f.Namespace.Name).List(metav1.ListOptions{}) pods, err := clientSet.CoreV1().Pods(f.Namespace.Name).List(metav1.ListOptions{})
if err != nil { if err != nil {
@ -445,17 +446,13 @@ var _ = SIGDescribe("Garbage collector", func() {
framework.Failf("%v", err) framework.Failf("%v", err)
} }
By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods") By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods")
if err := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { time.Sleep(30 * time.Second)
pods, err := podClient.List(metav1.ListOptions{}) pods, err := podClient.List(metav1.ListOptions{})
if err != nil { if err != nil {
return false, fmt.Errorf("Failed to list pods: %v", err) framework.Failf("Failed to list pods: %v", err)
} }
if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a { if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a {
return false, fmt.Errorf("expect %d pods, got %d pods", e, a) framework.Failf("expect %d pods, got %d pods", e, a)
}
return false, nil
}); err != nil && err != wait.ErrWaitTimeout {
framework.Failf("%v", err)
} }
gatherMetrics(f) gatherMetrics(f)
}) })
@ -494,17 +491,13 @@ var _ = SIGDescribe("Garbage collector", func() {
framework.Failf("failed to delete the rc: %v", err) framework.Failf("failed to delete the rc: %v", err)
} }
By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods") By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods")
if err := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { time.Sleep(30 * time.Second)
pods, err := podClient.List(metav1.ListOptions{}) pods, err := podClient.List(metav1.ListOptions{})
if err != nil { if err != nil {
return false, fmt.Errorf("Failed to list pods: %v", err) framework.Failf("Failed to list pods: %v", err)
} }
if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a { if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a {
return false, fmt.Errorf("expect %d pods, got %d pods", e, a) framework.Failf("expect %d pods, got %d pods", e, a)
}
return false, nil
}); err != nil && err != wait.ErrWaitTimeout {
framework.Failf("%v", err)
} }
gatherMetrics(f) gatherMetrics(f)
}) })
@ -552,14 +545,17 @@ var _ = SIGDescribe("Garbage collector", func() {
err = wait.PollImmediate(500*time.Millisecond, 1*time.Minute, func() (bool, error) { err = wait.PollImmediate(500*time.Millisecond, 1*time.Minute, func() (bool, error) {
return verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 0, 0) return verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 0, 0)
}) })
if err == wait.ErrWaitTimeout { if err != nil {
err = fmt.Errorf("Failed to wait for all rs to be garbage collected: %v", err) errList := make([]error, 0)
errList = append(errList, err)
remainingRSs, err := rsClient.List(metav1.ListOptions{}) remainingRSs, err := rsClient.List(metav1.ListOptions{})
if err != nil { if err != nil {
framework.Failf("failed to list RSs post mortem: %v", err) errList = append(errList, fmt.Errorf("failed to list RSs post mortem: %v", err))
} else { } else {
framework.Failf("remaining rs are: %#v", remainingRSs) errList = append(errList, fmt.Errorf("remaining rs are: %#v", remainingRSs))
} }
aggregatedError := utilerrors.NewAggregate(errList)
framework.Failf("Failed to wait for all rs to be garbage collected: %v", aggregatedError)
} }
@ -605,24 +601,28 @@ var _ = SIGDescribe("Garbage collector", func() {
if err := deployClient.Delete(deployment.ObjectMeta.Name, deleteOptions); err != nil { if err := deployClient.Delete(deployment.ObjectMeta.Name, deleteOptions); err != nil {
framework.Failf("failed to delete the deployment: %v", err) framework.Failf("failed to delete the deployment: %v", err)
} }
By("wait for 2 Minute to see if the garbage collector mistakenly deletes the rs") By("wait for 30 seconds to see if the garbage collector mistakenly deletes the rs")
err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { time.Sleep(30 * time.Second)
return verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 1, 2) ok, err := verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 1, 2)
})
if err != nil { if err != nil {
err = fmt.Errorf("Failed to wait to see if the garbage collecter mistakenly deletes the rs: %v", err) framework.Failf("Unexpected error while verifying remaining deployments, rs, and pods: %v", err)
}
if !ok {
errList := make([]error, 0)
remainingRSs, err := rsClient.List(metav1.ListOptions{}) remainingRSs, err := rsClient.List(metav1.ListOptions{})
if err != nil { if err != nil {
framework.Failf("failed to list RSs post mortem: %v", err) errList = append(errList, fmt.Errorf("failed to list RSs post mortem: %v", err))
} else { } else {
framework.Failf("remaining rs post mortem: %#v", remainingRSs) errList = append(errList, fmt.Errorf("remaining rs post mortem: %#v", remainingRSs))
} }
remainingDSs, err := deployClient.List(metav1.ListOptions{}) remainingDSs, err := deployClient.List(metav1.ListOptions{})
if err != nil { if err != nil {
framework.Failf("failed to list Deployments post mortem: %v", err) errList = append(errList, fmt.Errorf("failed to list Deployments post mortem: %v", err))
} else { } else {
framework.Failf("remaining deployment's post mortem: %#v", remainingDSs) errList = append(errList, fmt.Errorf("remaining deployment's post mortem: %#v", remainingDSs))
} }
aggregatedError := utilerrors.NewAggregate(errList)
framework.Failf("Failed to verify remaining deployments, rs, and pods: %v", aggregatedError)
} }
rs, err := clientSet.ExtensionsV1beta1().ReplicaSets(f.Namespace.Name).List(metav1.ListOptions{}) rs, err := clientSet.ExtensionsV1beta1().ReplicaSets(f.Namespace.Name).List(metav1.ListOptions{})
if err != nil { if err != nil {