From 21bc54508b24b1cc20931d0b219f8e892f198772 Mon Sep 17 00:00:00 2001 From: Jun Xiang Tee Date: Fri, 27 Oct 2017 13:19:50 -0700 Subject: [PATCH] convert testFailedDeployment e2e test to integration test --- test/e2e/apps/deployment.go | 43 ----------------- test/e2e/framework/deployment_util.go | 35 +------------- .../integration/deployment/deployment_test.go | 47 +++++++++++++++++++ test/integration/deployment/util.go | 8 ++++ test/utils/deployment.go | 39 +++++++++++++++ 5 files changed, 96 insertions(+), 76 deletions(-) diff --git a/test/e2e/apps/deployment.go b/test/e2e/apps/deployment.go index 147adac8916..a17234c2452 100644 --- a/test/e2e/apps/deployment.go +++ b/test/e2e/apps/deployment.go @@ -93,9 +93,6 @@ var _ = SIGDescribe("Deployment", func() { It("overlapping deployment should not fight with each other", func() { testOverlappingDeployment(f) }) - It("lack of progress should be reported in the deployment status", func() { - testFailedDeployment(f) - }) It("iterative rollouts should eventually progress", func() { testIterativeDeployments(f) }) @@ -816,46 +813,6 @@ func testOverlappingDeployment(f *framework.Framework) { Expect(rsList.Items).To(HaveLen(2)) } -func testFailedDeployment(f *framework.Framework) { - ns := f.Namespace.Name - c := f.ClientSet - - podLabels := map[string]string{"name": NginxImageName} - replicas := int32(1) - - // Create a nginx deployment. - deploymentName := "progress-check" - nonExistentImage := "nginx:not-there" - ten := int32(10) - d := framework.NewDeployment(deploymentName, replicas, podLabels, NginxImageName, nonExistentImage, extensions.RecreateDeploymentStrategyType) - d.Spec.ProgressDeadlineSeconds = &ten - - framework.Logf("Creating deployment %q with progressDeadlineSeconds set to %ds and a non-existent image", deploymentName, ten) - deployment, err := c.Extensions().Deployments(ns).Create(d) - Expect(err).NotTo(HaveOccurred()) - - framework.Logf("Waiting for deployment %q new replica set to come up", deploymentName) - Expect(framework.WaitForDeploymentUpdatedReplicasLTE(c, ns, deploymentName, replicas, deployment.Generation)) - - framework.Logf("Checking deployment %q for a timeout condition", deploymentName) - Expect(framework.WaitForDeploymentWithCondition(c, ns, deploymentName, deploymentutil.TimedOutReason, extensions.DeploymentProgressing)).NotTo(HaveOccurred()) - - framework.Logf("Updating deployment %q with a good image", deploymentName) - deployment, err = framework.UpdateDeploymentWithRetries(c, ns, deployment.Name, func(update *extensions.Deployment) { - update.Spec.Template.Spec.Containers[0].Image = NginxImage - }) - Expect(err).NotTo(HaveOccurred()) - - framework.Logf("Waiting for deployment %q new replica set to come up", deploymentName) - Expect(framework.WaitForDeploymentUpdatedReplicasLTE(c, ns, deploymentName, replicas, deployment.Generation)) - - framework.Logf("Waiting for deployment %q status", deploymentName) - Expect(framework.WaitForDeploymentComplete(c, deployment)).NotTo(HaveOccurred()) - - framework.Logf("Checking deployment %q for a complete condition", deploymentName) - Expect(framework.WaitForDeploymentWithCondition(c, ns, deploymentName, deploymentutil.NewRSAvailableReason, extensions.DeploymentProgressing)).NotTo(HaveOccurred()) -} - func randomScale(d *extensions.Deployment, i int) { switch r := rand.Float32(); { case r < 0.3: diff --git a/test/e2e/framework/deployment_util.go b/test/e2e/framework/deployment_util.go index a0755659d32..a18e3484444 100644 --- a/test/e2e/framework/deployment_util.go +++ b/test/e2e/framework/deployment_util.go @@ -72,25 +72,7 @@ func WaitForObservedDeployment(c clientset.Interface, ns, deploymentName string, } func WaitForDeploymentWithCondition(c clientset.Interface, ns, deploymentName, reason string, condType extensions.DeploymentConditionType) error { - var deployment *extensions.Deployment - pollErr := wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) { - d, err := c.Extensions().Deployments(ns).Get(deploymentName, metav1.GetOptions{}) - if err != nil { - return false, err - } - deployment = d - cond := deploymentutil.GetDeploymentCondition(deployment.Status, condType) - return cond != nil && cond.Reason == reason, nil - }) - if pollErr == wait.ErrWaitTimeout { - pollErr = fmt.Errorf("deployment %q never updated with the desired condition and reason: %v", deployment.Name, deployment.Status.Conditions) - _, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(deployment, c.ExtensionsV1beta1()) - if err == nil { - logReplicaSetsOfDeployment(deployment, allOldRSs, newRS) - logPodsOfDeployment(c, deployment, append(allOldRSs, newRS)) - } - } - return pollErr + return testutils.WaitForDeploymentWithCondition(c, ns, deploymentName, reason, condType, Logf, Poll, pollLongTimeout) } // WaitForDeploymentRevisionAndImage waits for the deployment's and its new RS's revision and container image to match the given revision and image. @@ -145,20 +127,7 @@ func WaitForDeploymentCompleteAndCheckRolling(c clientset.Interface, d *extensio // WaitForDeploymentUpdatedReplicasLTE waits for given deployment to be observed by the controller and has at least a number of updatedReplicas func WaitForDeploymentUpdatedReplicasLTE(c clientset.Interface, ns, deploymentName string, minUpdatedReplicas int32, desiredGeneration int64) error { - err := wait.Poll(Poll, 5*time.Minute, func() (bool, error) { - deployment, err := c.Extensions().Deployments(ns).Get(deploymentName, metav1.GetOptions{}) - if err != nil { - return false, err - } - if deployment.Status.ObservedGeneration >= desiredGeneration && deployment.Status.UpdatedReplicas >= minUpdatedReplicas { - return true, nil - } - return false, nil - }) - if err != nil { - return fmt.Errorf("error waiting for deployment %s to have at least %d updpatedReplicas: %v", deploymentName, minUpdatedReplicas, err) - } - return nil + return testutils.WaitForDeploymentUpdatedReplicasLTE(c, ns, deploymentName, minUpdatedReplicas, desiredGeneration, Poll, pollLongTimeout) } // WaitForDeploymentRollbackCleared waits for given deployment either started rolling back or doesn't need to rollback. diff --git a/test/integration/deployment/deployment_test.go b/test/integration/deployment/deployment_test.go index 6e228511b3b..7eb34cfa8d5 100644 --- a/test/integration/deployment/deployment_test.go +++ b/test/integration/deployment/deployment_test.go @@ -633,3 +633,50 @@ func TestDeploymentLabelAdopted(t *testing.T) { t.Errorf("found mismatching pod-template-hash value: rs hash = %s whereas pod hash = %s", rsHash, podHash) } } + +// Deployment should have a timeout condition when it fails to progress after given deadline. +func TestFailedDeployment(t *testing.T) { + s, closeFn, rm, dc, informers, c := dcSetup(t) + defer closeFn() + name := "test-failed-deployment" + ns := framework.CreateTestingNamespace(name, s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + deploymentName := "progress-check" + replicas := int32(1) + three := int32(3) + tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)} + tester.deployment.Spec.ProgressDeadlineSeconds = &three + var err error + tester.deployment, err = c.ExtensionsV1beta1().Deployments(ns.Name).Create(tester.deployment) + if err != nil { + t.Fatalf("failed to create deployment %q: %v", deploymentName, err) + } + + // Start informer and controllers + stopCh := make(chan struct{}) + defer close(stopCh) + informers.Start(stopCh) + go rm.Run(5, stopCh) + go dc.Run(5, stopCh) + + if err = tester.waitForDeploymentUpdatedReplicasLTE(replicas); err != nil { + t.Fatal(err) + } + + // Pods are not marked as Ready, therefore the deployment progress will eventually timeout after progressDeadlineSeconds has passed. + // Wait for the deployment to have a progress timeout condition. + if err = tester.waitForDeploymentWithCondition(deploymentutil.TimedOutReason, v1beta1.DeploymentProgressing); err != nil { + t.Fatal(err) + } + + // Manually mark pods as Ready and wait for deployment to complete. + if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil { + t.Fatalf("deployment %q fails to have its status becoming valid: %v", deploymentName, err) + } + + // Wait for the deployment to have a progress complete condition. + if err = tester.waitForDeploymentWithCondition(deploymentutil.NewRSAvailableReason, v1beta1.DeploymentProgressing); err != nil { + t.Fatal(err) + } +} diff --git a/test/integration/deployment/util.go b/test/integration/deployment/util.go index 7d771d1a759..dde774db070 100644 --- a/test/integration/deployment/util.go +++ b/test/integration/deployment/util.go @@ -343,3 +343,11 @@ func (d *deploymentTester) waitForDeploymentRollbackCleared() error { func (d *deploymentTester) checkDeploymentRevisionAndImage(revision, image string) error { return testutil.CheckDeploymentRevisionAndImage(d.c, d.deployment.Namespace, d.deployment.Name, revision, image) } + +func (d *deploymentTester) waitForDeploymentUpdatedReplicasLTE(minUpdatedReplicas int32) error { + return testutil.WaitForDeploymentUpdatedReplicasLTE(d.c, d.deployment.Namespace, d.deployment.Name, minUpdatedReplicas, d.deployment.Generation, pollInterval, pollTimeout) +} + +func (d *deploymentTester) waitForDeploymentWithCondition(reason string, condType v1beta1.DeploymentConditionType) error { + return testutil.WaitForDeploymentWithCondition(d.c, d.deployment.Namespace, d.deployment.Name, reason, condType, d.t.Logf, pollInterval, pollTimeout) +} diff --git a/test/utils/deployment.go b/test/utils/deployment.go index 896a610fea4..f57de2a7f49 100644 --- a/test/utils/deployment.go +++ b/test/utils/deployment.go @@ -301,3 +301,42 @@ func WaitForDeploymentRollbackCleared(c clientset.Interface, ns, deploymentName } return nil } + +// WaitForDeploymentUpdatedReplicasLTE waits for given deployment to be observed by the controller and has at least a number of updatedReplicas +func WaitForDeploymentUpdatedReplicasLTE(c clientset.Interface, ns, deploymentName string, minUpdatedReplicas int32, desiredGeneration int64, pollInterval, pollTimeout time.Duration) error { + var deployment *extensions.Deployment + err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { + d, err := c.ExtensionsV1beta1().Deployments(ns).Get(deploymentName, metav1.GetOptions{}) + if err != nil { + return false, err + } + deployment = d + return deployment.Status.ObservedGeneration >= desiredGeneration && deployment.Status.UpdatedReplicas >= minUpdatedReplicas, nil + }) + if err != nil { + return fmt.Errorf("error waiting for deployment %q to have at least %d updatedReplicas: %v; latest .status.updatedReplicas: %d", deploymentName, minUpdatedReplicas, err, deployment.Status.UpdatedReplicas) + } + return nil +} + +func WaitForDeploymentWithCondition(c clientset.Interface, ns, deploymentName, reason string, condType extensions.DeploymentConditionType, logf LogfFn, pollInterval, pollTimeout time.Duration) error { + var deployment *extensions.Deployment + pollErr := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { + d, err := c.ExtensionsV1beta1().Deployments(ns).Get(deploymentName, metav1.GetOptions{}) + if err != nil { + return false, err + } + deployment = d + cond := deploymentutil.GetDeploymentCondition(deployment.Status, condType) + return cond != nil && cond.Reason == reason, nil + }) + if pollErr == wait.ErrWaitTimeout { + pollErr = fmt.Errorf("deployment %q never updated with the desired condition and reason, latest deployment conditions: %+v", deployment.Name, deployment.Status.Conditions) + _, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(deployment, c.ExtensionsV1beta1()) + if err == nil { + LogReplicaSetsOfDeployment(deployment, allOldRSs, newRS, logf) + LogPodsOfDeployment(c, deployment, append(allOldRSs, newRS), logf) + } + } + return pollErr +}