From e6c60a746595dc6c0d1b7ec2df855c005cf57247 Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Tue, 27 Feb 2018 20:40:31 +0100 Subject: [PATCH] Add retries to resource deletions in testing framework --- test/e2e/framework/util.go | 42 ++------------- test/e2e/scalability/density.go | 3 +- test/e2e/scalability/load.go | 2 +- test/utils/BUILD | 2 + test/utils/delete_resources.go | 95 +++++++++++++++++++++++++++++++++ test/utils/runners.go | 4 +- 6 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 test/utils/delete_resources.go diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 456133a9f7f..a334a71397e 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -2923,23 +2923,6 @@ func getRuntimeObjectForKind(c clientset.Interface, kind schema.GroupKind, ns, n } } -func deleteResource(c clientset.Interface, kind schema.GroupKind, ns, name string, deleteOption *metav1.DeleteOptions) error { - switch kind { - case api.Kind("ReplicationController"): - return c.CoreV1().ReplicationControllers(ns).Delete(name, deleteOption) - case extensionsinternal.Kind("ReplicaSet"), appsinternal.Kind("ReplicaSet"): - return c.ExtensionsV1beta1().ReplicaSets(ns).Delete(name, deleteOption) - case extensionsinternal.Kind("Deployment"), appsinternal.Kind("Deployment"): - return c.ExtensionsV1beta1().Deployments(ns).Delete(name, deleteOption) - case extensionsinternal.Kind("DaemonSet"): - return c.ExtensionsV1beta1().DaemonSets(ns).Delete(name, deleteOption) - case batchinternal.Kind("Job"): - return c.BatchV1().Jobs(ns).Delete(name, deleteOption) - default: - return fmt.Errorf("Unsupported kind when deleting: %v", kind) - } -} - func getSelectorFromRuntimeObject(obj runtime.Object) (labels.Selector, error) { switch typed := obj.(type) { case *v1.ReplicationController: @@ -2986,10 +2969,6 @@ func getReplicasFromRuntimeObject(obj runtime.Object) (int32, error) { } } -func getReaperForKind(internalClientset internalclientset.Interface, kind schema.GroupKind) (kubectl.Reaper, error) { - return kubectl.ReaperFor(kind, internalClientset) -} - // DeleteResourceAndPods deletes a given resource and all pods it spawned func DeleteResourceAndPods(clientset clientset.Interface, internalClientset internalclientset.Interface, kind schema.GroupKind, ns, name string) error { By(fmt.Sprintf("deleting %v %s in namespace %s", kind, name, ns)) @@ -3006,23 +2985,13 @@ func DeleteResourceAndPods(clientset clientset.Interface, internalClientset inte if err != nil { return err } - reaper, err := getReaperForKind(internalClientset, kind) - if err != nil { - return err - } - ps, err := podStoreForSelector(clientset, ns, selector) if err != nil { return err } defer ps.Stop() startTime := time.Now() - err = reaper.Stop(ns, name, 0, nil) - if apierrs.IsNotFound(err) { - Logf("%v %s was already deleted: %v", kind, name, err) - return nil - } - if err != nil { + if err := testutils.DeleteResourceUsingReaperWithRetries(internalClientset, kind, ns, name, nil); err != nil { return fmt.Errorf("error while stopping %v: %s: %v", kind, name, err) } deleteTime := time.Now().Sub(startTime) @@ -3071,15 +3040,10 @@ func DeleteResourceAndWaitForGC(c clientset.Interface, kind schema.GroupKind, ns } defer ps.Stop() - startTime := time.Now() falseVar := false deleteOption := &metav1.DeleteOptions{OrphanDependents: &falseVar} - err = deleteResource(c, kind, ns, name, deleteOption) - if err != nil && apierrs.IsNotFound(err) { - Logf("%v %s was already deleted: %v", kind, name, err) - return nil - } - if err != nil { + startTime := time.Now() + if err := testutils.DeleteResourceWithRetries(c, kind, ns, name, deleteOption); err != nil { return err } deleteTime := time.Now().Sub(startTime) diff --git a/test/e2e/scalability/density.go b/test/e2e/scalability/density.go index 567392f5be8..469c3c8ad8e 100644 --- a/test/e2e/scalability/density.go +++ b/test/e2e/scalability/density.go @@ -538,8 +538,7 @@ var _ = SIGDescribe("Density", func() { namespaces, err := CreateNamespaces(f, numberOfCollections, fmt.Sprintf("density-%v", testArg.podsPerNode), testPhaseDurations.StartPhase(200, "namespace creation")) framework.ExpectNoError(err) if itArg.quotas { - err := CreateQuotas(f, namespaces, totalPods+nodeCount, testPhaseDurations.StartPhase(210, "quota creation")) - framework.ExpectNoError(err) + framework.ExpectNoError(CreateQuotas(f, namespaces, totalPods+nodeCount, testPhaseDurations.StartPhase(210, "quota creation"))) } configs := make([]testutils.RunObjectConfig, numberOfCollections) diff --git a/test/e2e/scalability/load.go b/test/e2e/scalability/load.go index 38bf5970c57..a717929c8c0 100644 --- a/test/e2e/scalability/load.go +++ b/test/e2e/scalability/load.go @@ -248,7 +248,7 @@ var _ = SIGDescribe("Load capacity", func() { framework.Logf("Starting to delete services...") deleteService := func(i int) { defer GinkgoRecover() - framework.ExpectNoError(clientset.CoreV1().Services(services[i].Namespace).Delete(services[i].Name, nil)) + framework.ExpectNoError(testutils.DeleteResourceWithRetries(clientset, api.Kind("Service"), services[i].Namespace, services[i].Name, nil)) } workqueue.Parallelize(serviceOperationsParallelism, len(services), deleteService) framework.Logf("Services deleted") diff --git a/test/utils/BUILD b/test/utils/BUILD index 40b9aa02b15..2c04822316f 100644 --- a/test/utils/BUILD +++ b/test/utils/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "conditions.go", "create_resources.go", + "delete_resources.go", "density_utils.go", "deployment.go", "pod_store.go", @@ -21,6 +22,7 @@ go_library( importpath = "k8s.io/kubernetes/test/utils", deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/apis/apps:go_default_library", "//pkg/apis/batch:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/extensions:go_default_library", diff --git a/test/utils/delete_resources.go b/test/utils/delete_resources.go new file mode 100644 index 00000000000..de5771c9503 --- /dev/null +++ b/test/utils/delete_resources.go @@ -0,0 +1,95 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO: Refactor common part of functions in this file for generic object kinds. + +package utils + +import ( + "fmt" + + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + clientset "k8s.io/client-go/kubernetes" + appsinternal "k8s.io/kubernetes/pkg/apis/apps" + batchinternal "k8s.io/kubernetes/pkg/apis/batch" + api "k8s.io/kubernetes/pkg/apis/core" + extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/kubectl" +) + +func deleteResource(c clientset.Interface, kind schema.GroupKind, namespace, name string, options *metav1.DeleteOptions) error { + switch kind { + case api.Kind("Pod"): + return c.CoreV1().Pods(namespace).Delete(name, options) + case api.Kind("ReplicationController"): + return c.CoreV1().ReplicationControllers(namespace).Delete(name, options) + case extensionsinternal.Kind("ReplicaSet"), appsinternal.Kind("ReplicaSet"): + return c.ExtensionsV1beta1().ReplicaSets(namespace).Delete(name, options) + case extensionsinternal.Kind("Deployment"), appsinternal.Kind("Deployment"): + return c.ExtensionsV1beta1().Deployments(namespace).Delete(name, options) + case extensionsinternal.Kind("DaemonSet"): + return c.ExtensionsV1beta1().DaemonSets(namespace).Delete(name, options) + case batchinternal.Kind("Job"): + return c.BatchV1().Jobs(namespace).Delete(name, options) + case api.Kind("Secret"): + return c.CoreV1().Secrets(namespace).Delete(name, options) + case api.Kind("ConfigMap"): + return c.CoreV1().ConfigMaps(namespace).Delete(name, options) + case api.Kind("Service"): + return c.CoreV1().Services(namespace).Delete(name, options) + default: + return fmt.Errorf("Unsupported kind when deleting: %v", kind) + } +} + +func getReaperForKind(c internalclientset.Interface, kind schema.GroupKind) (kubectl.Reaper, error) { + return kubectl.ReaperFor(kind, c) +} + +func DeleteResourceWithRetries(c clientset.Interface, kind schema.GroupKind, namespace, name string, options *metav1.DeleteOptions) error { + deleteFunc := func() (bool, error) { + err := deleteResource(c, kind, namespace, name, options) + if err == nil || apierrs.IsNotFound(err) { + return true, nil + } + if IsRetryableAPIError(err) { + return false, nil + } + return false, fmt.Errorf("Failed to delete object with non-retriable error: %v", err) + } + return RetryWithExponentialBackOff(deleteFunc) +} + +func DeleteResourceUsingReaperWithRetries(c internalclientset.Interface, kind schema.GroupKind, namespace, name string, options *metav1.DeleteOptions) error { + reaper, err := getReaperForKind(c, kind) + if err != nil { + return err + } + deleteFunc := func() (bool, error) { + err := reaper.Stop(namespace, name, 0, options) + if err == nil || apierrs.IsNotFound(err) { + return true, nil + } + if IsRetryableAPIError(err) { + return false, nil + } + return false, fmt.Errorf("Failed to delete object with non-retriable error: %v", err) + } + return RetryWithExponentialBackOff(deleteFunc) +} diff --git a/test/utils/runners.go b/test/utils/runners.go index 43d77bf79ca..638dc2cb2a4 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -1113,7 +1113,7 @@ func (config *SecretConfig) Run() error { } func (config *SecretConfig) Stop() error { - if err := config.Client.CoreV1().Secrets(config.Namespace).Delete(config.Name, &metav1.DeleteOptions{}); err != nil { + if err := DeleteResourceWithRetries(config.Client, api.Kind("Secret"), config.Namespace, config.Name, &metav1.DeleteOptions{}); err != nil { return fmt.Errorf("Error deleting secret: %v", err) } config.LogFunc("Deleted secret %v/%v", config.Namespace, config.Name) @@ -1171,7 +1171,7 @@ func (config *ConfigMapConfig) Run() error { } func (config *ConfigMapConfig) Stop() error { - if err := config.Client.CoreV1().ConfigMaps(config.Namespace).Delete(config.Name, &metav1.DeleteOptions{}); err != nil { + if err := DeleteResourceWithRetries(config.Client, api.Kind("ConfigMap"), config.Namespace, config.Name, &metav1.DeleteOptions{}); err != nil { return fmt.Errorf("Error deleting configmap: %v", err) } config.LogFunc("Deleted configmap %v/%v", config.Namespace, config.Name)