diff --git a/pkg/apis/policy/v1beta1/types.go b/pkg/apis/policy/v1beta1/types.go index 99409203943..381daaa1e46 100644 --- a/pkg/apis/policy/v1beta1/types.go +++ b/pkg/apis/policy/v1beta1/types.go @@ -89,6 +89,9 @@ type PodDisruptionBudgetList struct { Items []PodDisruptionBudget `json:"items" protobuf:"bytes,2,rep,name=items"` } +// +genclient=true +// +noMethods=true + // Eviction evicts a pod from its node subject to certain policies and safety constraints. // This is a subresource of Pod. A request to cause such an eviction is // created by POSTing to .../pods//evictions. diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/BUILD b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/BUILD index 2c5bc912bdb..abe54dedc90 100644 --- a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/BUILD +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/BUILD @@ -14,6 +14,8 @@ go_library( name = "go_default_library", srcs = [ "doc.go", + "eviction.go", + "eviction_expansion.go", "generated_expansion.go", "poddisruptionbudget.go", "policy_client.go", diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction.go new file mode 100644 index 00000000000..66cf3cdcc01 --- /dev/null +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction.go @@ -0,0 +1,46 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + restclient "k8s.io/kubernetes/pkg/client/restclient" +) + +// EvictionsGetter has a method to return a EvictionInterface. +// A group's client should implement this interface. +type EvictionsGetter interface { + Evictions(namespace string) EvictionInterface +} + +// EvictionInterface has methods to work with Eviction resources. +type EvictionInterface interface { + EvictionExpansion +} + +// evictions implements EvictionInterface +type evictions struct { + client restclient.Interface + ns string +} + +// newEvictions returns a Evictions +func newEvictions(c *PolicyV1beta1Client, namespace string) *evictions { + return &evictions{ + client: c.RESTClient(), + ns: namespace, + } +} diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction_expansion.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction_expansion.go new file mode 100644 index 00000000000..c24252cb015 --- /dev/null +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction_expansion.go @@ -0,0 +1,38 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1" +) + +// The EvictionExpansion interface allows manually adding extra methods to the ScaleInterface. +type EvictionExpansion interface { + Evict(eviction *policy.Eviction) error +} + +func (c *evictions) Evict(eviction *policy.Eviction) error { + return c.client.Post(). + AbsPath("/api/v1"). + Namespace(eviction.Namespace). + Resource("pods"). + Name(eviction.Name). + SubResource("eviction"). + Body(eviction). + Do(). + Error() +} diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/BUILD b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/BUILD index f9427fe0a53..4b7e6471246 100644 --- a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/BUILD +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/BUILD @@ -14,6 +14,8 @@ go_library( name = "go_default_library", srcs = [ "doc.go", + "fake_eviction.go", + "fake_eviction_expansion.go", "fake_poddisruptionbudget.go", "fake_policy_client.go", ], diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction.go new file mode 100644 index 00000000000..d6f1796400b --- /dev/null +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction.go @@ -0,0 +1,23 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +// FakeEvictions implements EvictionInterface +type FakeEvictions struct { + Fake *FakePolicyV1beta1 + ns string +} diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction_expansion.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction_expansion.go new file mode 100644 index 00000000000..60c0c2964ce --- /dev/null +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction_expansion.go @@ -0,0 +1,33 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1" + core "k8s.io/kubernetes/pkg/client/testing/core" + "k8s.io/kubernetes/pkg/runtime/schema" +) + +func (c *FakeEvictions) Evict(eviction *policy.Eviction) error { + action := core.GetActionImpl{} + action.Verb = "post" + action.Namespace = c.ns + action.Resource = schema.GroupVersionResource{Group: "", Version: "", Resource: "pods"} + action.Subresource = "eviction" + _, err := c.Fake.Invokes(action, eviction) + return err +} diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_policy_client.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_policy_client.go index ef7bdf9857c..420b8bc1441 100644 --- a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_policy_client.go +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_policy_client.go @@ -26,6 +26,10 @@ type FakePolicyV1beta1 struct { *core.Fake } +func (c *FakePolicyV1beta1) Evictions(namespace string) v1beta1.EvictionInterface { + return &FakeEvictions{c, namespace} +} + func (c *FakePolicyV1beta1) PodDisruptionBudgets(namespace string) v1beta1.PodDisruptionBudgetInterface { return &FakePodDisruptionBudgets{c, namespace} } diff --git a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/policy_client.go b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/policy_client.go index c827a2a0b6b..a0d9640c4cb 100644 --- a/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/policy_client.go +++ b/pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/policy_client.go @@ -27,6 +27,7 @@ import ( type PolicyV1beta1Interface interface { RESTClient() restclient.Interface + EvictionsGetter PodDisruptionBudgetsGetter } @@ -35,6 +36,10 @@ type PolicyV1beta1Client struct { restClient restclient.Interface } +func (c *PolicyV1beta1Client) Evictions(namespace string) EvictionInterface { + return newEvictions(c, namespace) +} + func (c *PolicyV1beta1Client) PodDisruptionBudgets(namespace string) PodDisruptionBudgetInterface { return newPodDisruptionBudgets(c, namespace) } diff --git a/pkg/client/listers/policy/v1beta1/BUILD b/pkg/client/listers/policy/v1beta1/BUILD index 435fed14ccd..9fa776fc477 100644 --- a/pkg/client/listers/policy/v1beta1/BUILD +++ b/pkg/client/listers/policy/v1beta1/BUILD @@ -13,6 +13,7 @@ load( go_library( name = "go_default_library", srcs = [ + "eviction.go", "expansion_generated.go", "poddisruptionbudget.go", ], diff --git a/pkg/client/listers/policy/v1beta1/eviction.go b/pkg/client/listers/policy/v1beta1/eviction.go new file mode 100644 index 00000000000..e45bc98cb06 --- /dev/null +++ b/pkg/client/listers/policy/v1beta1/eviction.go @@ -0,0 +1,95 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file was automatically generated by lister-gen with arguments: --input-dirs=[k8s.io/kubernetes/pkg/api,k8s.io/kubernetes/pkg/api/v1,k8s.io/kubernetes/pkg/apis/abac,k8s.io/kubernetes/pkg/apis/abac/v0,k8s.io/kubernetes/pkg/apis/abac/v1beta1,k8s.io/kubernetes/pkg/apis/apps,k8s.io/kubernetes/pkg/apis/apps/v1beta1,k8s.io/kubernetes/pkg/apis/authentication,k8s.io/kubernetes/pkg/apis/authentication/v1beta1,k8s.io/kubernetes/pkg/apis/authorization,k8s.io/kubernetes/pkg/apis/authorization/v1beta1,k8s.io/kubernetes/pkg/apis/autoscaling,k8s.io/kubernetes/pkg/apis/autoscaling/v1,k8s.io/kubernetes/pkg/apis/batch,k8s.io/kubernetes/pkg/apis/batch/v1,k8s.io/kubernetes/pkg/apis/batch/v2alpha1,k8s.io/kubernetes/pkg/apis/certificates,k8s.io/kubernetes/pkg/apis/certificates/v1alpha1,k8s.io/kubernetes/pkg/apis/componentconfig,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,k8s.io/kubernetes/pkg/apis/extensions,k8s.io/kubernetes/pkg/apis/extensions/v1beta1,k8s.io/kubernetes/pkg/apis/imagepolicy,k8s.io/kubernetes/pkg/apis/imagepolicy/v1alpha1,k8s.io/kubernetes/pkg/apis/meta/v1,k8s.io/kubernetes/pkg/apis/policy,k8s.io/kubernetes/pkg/apis/policy/v1alpha1,k8s.io/kubernetes/pkg/apis/policy/v1beta1,k8s.io/kubernetes/pkg/apis/rbac,k8s.io/kubernetes/pkg/apis/rbac/v1alpha1,k8s.io/kubernetes/pkg/apis/storage,k8s.io/kubernetes/pkg/apis/storage/v1beta1] + +package v1beta1 + +import ( + "k8s.io/kubernetes/pkg/api/errors" + policy "k8s.io/kubernetes/pkg/apis/policy" + v1beta1 "k8s.io/kubernetes/pkg/apis/policy/v1beta1" + "k8s.io/kubernetes/pkg/client/cache" + "k8s.io/kubernetes/pkg/labels" +) + +// EvictionLister helps list Evictions. +type EvictionLister interface { + // List lists all Evictions in the indexer. + List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) + // Evictions returns an object that can list and get Evictions. + Evictions(namespace string) EvictionNamespaceLister + EvictionListerExpansion +} + +// evictionLister implements the EvictionLister interface. +type evictionLister struct { + indexer cache.Indexer +} + +// NewEvictionLister returns a new EvictionLister. +func NewEvictionLister(indexer cache.Indexer) EvictionLister { + return &evictionLister{indexer: indexer} +} + +// List lists all Evictions in the indexer. +func (s *evictionLister) List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1beta1.Eviction)) + }) + return ret, err +} + +// Evictions returns an object that can list and get Evictions. +func (s *evictionLister) Evictions(namespace string) EvictionNamespaceLister { + return evictionNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// EvictionNamespaceLister helps list and get Evictions. +type EvictionNamespaceLister interface { + // List lists all Evictions in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) + // Get retrieves the Eviction from the indexer for a given namespace and name. + Get(name string) (*v1beta1.Eviction, error) + EvictionNamespaceListerExpansion +} + +// evictionNamespaceLister implements the EvictionNamespaceLister +// interface. +type evictionNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all Evictions in the indexer for a given namespace. +func (s evictionNamespaceLister) List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1beta1.Eviction)) + }) + return ret, err +} + +// Get retrieves the Eviction from the indexer for a given namespace and name. +func (s evictionNamespaceLister) Get(name string) (*v1beta1.Eviction, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(policy.Resource("eviction"), name) + } + return obj.(*v1beta1.Eviction), nil +} diff --git a/pkg/client/listers/policy/v1beta1/expansion_generated.go b/pkg/client/listers/policy/v1beta1/expansion_generated.go index e3c6960406b..a7b64d1f10d 100644 --- a/pkg/client/listers/policy/v1beta1/expansion_generated.go +++ b/pkg/client/listers/policy/v1beta1/expansion_generated.go @@ -18,6 +18,14 @@ limitations under the License. package v1beta1 +// EvictionListerExpansion allows custom methods to be added to +// EvictionLister. +type EvictionListerExpansion interface{} + +// EvictionNamespaceListerExpansion allows custom methods to be added to +// EvictionNamespaeLister. +type EvictionNamespaceListerExpansion interface{} + // PodDisruptionBudgetListerExpansion allows custom methods to be added to // PodDisruptionBudgetLister. type PodDisruptionBudgetListerExpansion interface{} diff --git a/test/integration/evictions/evictions_test.go b/test/integration/evictions/evictions_test.go new file mode 100644 index 00000000000..840862656e4 --- /dev/null +++ b/test/integration/evictions/evictions_test.go @@ -0,0 +1,257 @@ +// +build integration,!no-etcd + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package evictions + +import ( + "fmt" + "net/http/httptest" + "testing" + "time" + + "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/api/v1" + metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/apis/policy/v1beta1" + "k8s.io/kubernetes/pkg/client/cache" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/controller/disruption" + "k8s.io/kubernetes/pkg/controller/informers" + "k8s.io/kubernetes/pkg/util/intstr" + "k8s.io/kubernetes/pkg/util/wait" + "k8s.io/kubernetes/test/integration/framework" +) + +const ( + defaultTimeout = 10 * time.Minute +) + +func newPod(podName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: podName, + Labels: map[string]string{"app": "test-evictions"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fake-name", + Image: "fakeimage", + }, + }, + }, + } +} + +func addPodConditionReady(pod *v1.Pod) { + pod.Status = v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + } +} + +func newPDB() *v1beta1.PodDisruptionBudget { + return &v1beta1.PodDisruptionBudget{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pdb", + }, + Spec: v1beta1.PodDisruptionBudgetSpec{ + MinAvailable: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test-evictions"}, + }, + }, + } +} + +func newEviction(ns, evictionName string, deleteOption *v1.DeleteOptions) *v1beta1.Eviction { + return &v1beta1.Eviction{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "Policy/v1beta1", + Kind: "Eviction", + }, + ObjectMeta: v1.ObjectMeta{ + Name: evictionName, + Namespace: ns, + }, + DeleteOptions: deleteOption, + } +} + +func rmSetup(t *testing.T) (*httptest.Server, *disruption.DisruptionController, cache.SharedIndexInformer, clientset.Interface) { + masterConfig := framework.NewIntegrationTestMasterConfig() + _, s := framework.RunAMaster(masterConfig) + + config := restclient.Config{Host: s.URL} + clientSet, err := clientset.NewForConfig(&config) + if err != nil { + t.Fatalf("Error in create clientset: %v", err) + } + resyncPeriod := 12 * time.Hour + informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), nil, resyncPeriod) + + rm := disruption.NewDisruptionController( + informers.Pods().Informer(), + clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")), + ) + return s, rm, informers.Pods().Informer(), clientSet +} + +func TestConcurrentEvictionRequests(t *testing.T) { + podNameFormat := "test-pod-%d" + + s, rm, podInformer, clientSet := rmSetup(t) + defer s.Close() + + ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + stopCh := make(chan struct{}) + go podInformer.Run(stopCh) + go rm.Run(stopCh) + + config := restclient.Config{Host: s.URL} + clientSet, err := clientset.NewForConfig(&config) + + var gracePeriodSeconds int64 = 30 + deleteOption := &v1.DeleteOptions{ + GracePeriodSeconds: &gracePeriodSeconds, + } + + // Generate 10 pods to evict + for i := 0; i < 10; i++ { + podName := fmt.Sprintf(podNameFormat, i) + pod := newPod(podName) + + if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil { + t.Errorf("Failed to create pod: %v", err) + } + + addPodConditionReady(pod) + if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil { + t.Fatal(err) + } + } + + waitToObservePods(t, podInformer, 10) + + pdb := newPDB() + if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil { + t.Errorf("Failed to create PodDisruptionBudget: %v", err) + } + + waitPDBStable(t, clientSet, 10, ns.Name, pdb.Name) + + doneCh := make(chan bool, 10) + errCh := make(chan error, 1) + // spawn 10 goroutine to concurrently evict the pods + for i := 0; i < 10; i++ { + go func(id int, doneCh chan bool, errCh chan error) { + evictionName := fmt.Sprintf(podNameFormat, id) + eviction := newEviction(ns.Name, evictionName, deleteOption) + + var e error + for { + e = clientSet.Policy().Evictions(ns.Name).Evict(eviction) + if errors.IsTooManyRequests(e) { + time.Sleep(5 * time.Second) + } else { + break + } + } + if e != nil { + if errors.IsConflict(err) { + fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e) + } else { + errCh <- e + } + return + } + doneCh <- true + }(i, doneCh, errCh) + } + + doneCount := 0 + for { + select { + case err := <-errCh: + t.Errorf("%v", err) + return + case <-doneCh: + doneCount++ + if doneCount == 10 { + return + } + case <-time.After(defaultTimeout): + t.Errorf("Eviction did not complete within %v", defaultTimeout) + } + } + + for i := 0; i < 10; i++ { + podName := fmt.Sprintf(podNameFormat, i) + _, err := clientSet.Core().Pods(ns.Name).Get(podName) + if !errors.IsNotFound(err) { + t.Errorf("Pod %q is expected to be evicted", podName) + } + } + + if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil { + t.Errorf("Failed to delete PodDisruptionBudget: %v", err) + } + + close(stopCh) +} + +// wait for the podInformer to observe the pods. Call this function before +// running the RS controller to prevent the rc manager from creating new pods +// rather than adopting the existing ones. +func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int) { + if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { + objects := podInformer.GetIndexer().List() + if len(objects) == podNum { + return true, nil + } + return false, nil + }); err != nil { + t.Fatal(err) + } +} + +func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) { + if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { + pdb, err := clientSet.Policy().PodDisruptionBudgets(ns).Get(pdbName) + if err != nil { + return false, err + } + if pdb.Status.CurrentHealthy != podNum { + return false, nil + } + return true, nil + }); err != nil { + t.Fatal(err) + } +}