In deployment rollback e2e test, wait for async actions to happen

test/e2e/deployment.go

@@ -602,7 +602,7 @@ func testRollbackDeployment(f *Framework) {
 	podName := "nginx"
 	deploymentPodLabels := map[string]string{"name": podName}
 
-	// Create a deployment to create nginx pods.
+	// 1. Create a deployment to create nginx pods.
 	deploymentName, deploymentImageName := "test-rollback-deployment", "nginx"
 	deploymentReplicas := 1
 	deploymentImage := "nginx"
@@ -613,26 +613,14 @@ func testRollbackDeployment(f *Framework) {
 	Expect(err).NotTo(HaveOccurred())
 	defer stopDeployment(c, f.Client, ns, deploymentName)
 
-	// Check that deployment is created fine.
+	// Wait for it to be updated to revision 1
-	deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", deploymentImage)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Verify that the required pods have come up.
+	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
-	err = verifyPods(unversionedClient, ns, "nginx", false, deploymentReplicas)
-	if err != nil {
-		Logf("error in waiting for pods to come up: %s", err)
-		Expect(err).NotTo(HaveOccurred())
-	}
-	deployment, err = c.Extensions().Deployments(ns).Get(deploymentName)
 	Expect(err).NotTo(HaveOccurred())
-	// DeploymentStatus should be appropriately updated.
-	Expect(deployment.Status.Replicas).Should(Equal(deploymentReplicas))
-	Expect(deployment.Status.UpdatedReplicas).Should(Equal(deploymentReplicas))
 
-	// Check if it's updated to revision 1 correctly
+	// 2. Update the deployment to create redis pods.
-	checkDeploymentRevision(c, ns, deploymentName, "1", deploymentImageName, deploymentImage)
-
-	// Update the deployment to create redis pods.
 	updatedDeploymentImage := "redis"
 	updatedDeploymentImageName := "redis"
 	_, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
@@ -641,37 +629,48 @@ func testRollbackDeployment(f *Framework) {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
+	// Wait for it to be updated to revision 2
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "2", updatedDeploymentImage)
+	Expect(err).NotTo(HaveOccurred())
+
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Check if it's updated to revision 2 correctly
+	// 3. Update the deploymentRollback to rollback to revision 1
-	checkDeploymentRevision(c, ns, deploymentName, "2", updatedDeploymentImageName, updatedDeploymentImage)
-
-	// Update the deploymentRollback to rollback to revision 1
 	revision := int64(1)
 	Logf("rolling back deployment %s to revision %d", deploymentName, revision)
 	rollback := newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
+	// Wait for the deployment to start rolling back
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
+	Expect(err).NotTo(HaveOccurred())
+	// TODO: report RollbackDone in deployment status and check it here
+
+	// Wait for it to be updated to revision 3
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "3", deploymentImage)
+	Expect(err).NotTo(HaveOccurred())
+
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Check if it's updated to revision 3 correctly
+	// 4. Update the deploymentRollback to rollback to last revision
-	checkDeploymentRevision(c, ns, deploymentName, "3", deploymentImageName, deploymentImage)
-
-	// Update the deploymentRollback to rollback to last revision
 	revision = 0
 	Logf("rolling back deployment %s to last revision", deploymentName)
 	rollback = newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Check if it's updated to revision 4 correctly
+	// Wait for it to be updated to revision 4
-	checkDeploymentRevision(c, ns, deploymentName, "4", updatedDeploymentImageName, updatedDeploymentImage)
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "4", updatedDeploymentImage)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
+	Expect(err).NotTo(HaveOccurred())
 }
 
 // testRollbackDeploymentRSNoRevision tests that deployment supports rollback even when there's old replica set without revision.
@@ -683,7 +682,6 @@ func testRollbackDeployment(f *Framework) {
 // TODO: When we finished reporting rollback status in deployment status, check the rollback status here in each case.
 func testRollbackDeploymentRSNoRevision(f *Framework) {
 	ns := f.Namespace.Name
-	unversionedClient := f.Client
 	c := clientset.FromUnversionedClient(f.Client)
 	podName := "nginx"
 	deploymentPodLabels := map[string]string{"name": podName}
@@ -692,6 +690,7 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
 		"pod":  "nginx",
 	}
 
+	// Create an old RS without revision
 	rsName := "test-rollback-no-revision-controller"
 	rsReplicas := 0
 	rs := newRS(rsName, rsReplicas, rsPodLabels, "nginx", "nginx")
@@ -700,7 +699,7 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
 	_, err := c.Extensions().ReplicaSets(ns).Create(rs)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Create a deployment to create nginx pods, which have different template than the replica set created above.
+	// 1. Create a deployment to create nginx pods, which have different template than the replica set created above.
 	deploymentName, deploymentImageName := "test-rollback-no-revision-deployment", "nginx"
 	deploymentReplicas := 1
 	deploymentImage := "nginx"
@@ -711,46 +710,36 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
 	Expect(err).NotTo(HaveOccurred())
 	defer stopDeployment(c, f.Client, ns, deploymentName)
 
-	// Check that deployment is created fine.
+	// Wait for it to be updated to revision 1
-	deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", deploymentImage)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Verify that the required pods have come up.
+	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
-	err = verifyPods(unversionedClient, ns, "nginx", false, deploymentReplicas)
-	if err != nil {
-		Logf("error in waiting for pods to come up: %s", err)
-		Expect(err).NotTo(HaveOccurred())
-	}
-	deployment, err = c.Extensions().Deployments(ns).Get(deploymentName)
 	Expect(err).NotTo(HaveOccurred())
-	// DeploymentStatus should be appropriately updated.
-	Expect(deployment.Status.Replicas).Should(Equal(deploymentReplicas))
-	Expect(deployment.Status.UpdatedReplicas).Should(Equal(deploymentReplicas))
-
-	// Check if it's updated to revision 1 correctly
-	checkDeploymentRevision(c, ns, deploymentName, "1", deploymentImageName, deploymentImage)
 
 	// Check that the replica set we created still doesn't contain revision information
 	rs, err = c.Extensions().ReplicaSets(ns).Get(rsName)
+	Expect(err).NotTo(HaveOccurred())
 	Expect(rs.Annotations[deploymentutil.RevisionAnnotation]).Should(Equal(""))
 
-	// Update the deploymentRollback to rollback to last revision
+	// 2. Update the deploymentRollback to rollback to last revision
-	// Since there's only 1 revision in history, it should stay as revision 1
+	//    Since there's only 1 revision in history, it should stay as revision 1
 	revision := int64(0)
 	Logf("rolling back deployment %s to last revision", deploymentName)
 	rollback := newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Wait until the rollback is done
+	// Wait for the deployment to start rolling back
-	waitForRollbackDone(c, deployment)
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
+	Expect(err).NotTo(HaveOccurred())
 	// TODO: report RollbackRevisionNotFound in deployment status and check it here
 
 	// The pod template shouldn't change since there's no last revision
 	// Check if the deployment is still revision 1 and still has the old pod template
 	checkDeploymentRevision(c, ns, deploymentName, "1", deploymentImageName, deploymentImage)
 
-	// Update the deployment to create redis pods.
+	// 3. Update the deployment to create redis pods.
 	updatedDeploymentImage := "redis"
 	updatedDeploymentImageName := "redis"
 	_, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
@@ -759,56 +748,61 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
+	// Wait for it to be updated to revision 2
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "2", updatedDeploymentImage)
+	Expect(err).NotTo(HaveOccurred())
+
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Check if it's updated to revision 2 correctly
+	// 4. Update the deploymentRollback to rollback to revision 1
-	checkDeploymentRevision(c, ns, deploymentName, "2", updatedDeploymentImageName, updatedDeploymentImage)
-
-	// Update the deploymentRollback to rollback to revision 1
 	revision = 1
 	Logf("rolling back deployment %s to revision %d", deploymentName, revision)
 	rollback = newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
+	// Wait for the deployment to start rolling back
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
 	Expect(err).NotTo(HaveOccurred())
-
-	// Wait until the rollback is done
-	waitForRollbackDone(c, deployment)
 	// TODO: report RollbackDone in deployment status and check it here
 
 	// The pod template should be updated to the one in revision 1
-	// Check if it's updated to revision 3 correctly
+	// Wait for it to be updated to revision 3
-	checkDeploymentRevision(c, ns, deploymentName, "3", deploymentImageName, deploymentImage)
+	err = waitForDeploymentRevisionAndImage(c, ns, deploymentName, "3", deploymentImage)
+	Expect(err).NotTo(HaveOccurred())
 
-	// Update the deploymentRollback to rollback to revision 10
+	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
-	// Since there's no revision 10 in history, it should stay as revision 3
+	Expect(err).NotTo(HaveOccurred())
+
+	// 5. Update the deploymentRollback to rollback to revision 10
+	//    Since there's no revision 10 in history, it should stay as revision 3
 	revision = 10
 	Logf("rolling back deployment %s to revision %d", deploymentName, revision)
 	rollback = newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Wait until the rollback is done
+	// Wait for the deployment to start rolling back
-	waitForRollbackDone(c, deployment)
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
+	Expect(err).NotTo(HaveOccurred())
 	// TODO: report RollbackRevisionNotFound in deployment status and check it here
 
 	// The pod template shouldn't change since there's no revision 10
 	// Check if it's still revision 3 and still has the old pod template
 	checkDeploymentRevision(c, ns, deploymentName, "3", deploymentImageName, deploymentImage)
 
-	// Update the deploymentRollback to rollback to revision 3
+	// 6. Update the deploymentRollback to rollback to revision 3
-	// Since it's already revision 3, it should be no-op
+	//    Since it's already revision 3, it should be no-op
 	revision = 3
 	Logf("rolling back deployment %s to revision %d", deploymentName, revision)
 	rollback = newDeploymentRollback(deploymentName, nil, revision)
 	err = c.Extensions().Deployments(ns).Rollback(rollback)
 	Expect(err).NotTo(HaveOccurred())
 
-	// Wait until the rollback is done
+	// Wait for the deployment to start rolling back
-	waitForRollbackDone(c, deployment)
+	err = waitForDeploymentRollbackCleared(c, ns, deploymentName)
+	Expect(err).NotTo(HaveOccurred())
 	// TODO: report RollbackTemplateUnchanged in deployment status and check it here
 
 	// The pod template shouldn't change since it's already revision 3

test/e2e/util.go

@@ -2296,7 +2296,62 @@ func waitForDeploymentStatus(c clientset.Interface, ns, deploymentName string, d
 		logReplicaSetsOfDeployment(deployment, allOldRSs, newRS)
 		logPodsOfReplicaSets(c, allRSs, minReadySeconds)
 	}
-	return err
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %s status to match expectation: %v", deploymentName, err)
+	}
+	return nil
+}
+
+// waitForDeploymentRollbackCleared waits for given deployment either started rolling back or doesn't need to rollback.
+// Note that rollback should be cleared shortly, so we only wait for 1 minute here to fail early.
+func waitForDeploymentRollbackCleared(c clientset.Interface, ns, deploymentName string) error {
+	err := wait.Poll(poll, 1*time.Minute, func() (bool, error) {
+		deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
+		if err != nil {
+			return false, err
+		}
+		// Rollback not set or is kicked off
+		if deployment.Spec.RollbackTo == nil {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %s rollbackTo to be cleared: %v", deploymentName, err)
+	}
+	return nil
+}
+
+// waitForDeploymentRevisionAndImage waits for the deployment's and its new RS's revision and container image to match the given revision and image.
+// Note that deployment revision and its new RS revision should be updated shortly, so we only wait for 1 minute here to fail early.
+func waitForDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName string, revision, image string) error {
+	var deployment *extensions.Deployment
+	var newRS *extensions.ReplicaSet
+	err := wait.Poll(poll, 1*time.Minute, func() (bool, error) {
+		var err error
+		deployment, err = c.Extensions().Deployments(ns).Get(deploymentName)
+		if err != nil {
+			return false, err
+		}
+		newRS, err = deploymentutil.GetNewReplicaSet(deployment, c)
+		if err != nil {
+			return false, err
+		}
+		// Check revision of this deployment, and of the new replica set of this deployment
+		if deployment.Annotations == nil || deployment.Annotations[deploymentutil.RevisionAnnotation] != revision ||
+			newRS.Annotations == nil || newRS.Annotations[deploymentutil.RevisionAnnotation] != revision ||
+			deployment.Spec.Template.Spec.Containers[0].Image != image || newRS.Spec.Template.Spec.Containers[0].Image != image {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		logReplicaSetsOfDeployment(deployment, nil, newRS)
+	}
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %s revision and image to match expectation: %v", deploymentName, err)
+	}
+	return nil
 }
 
 func waitForPodsReady(c *clientset.Clientset, ns, name string, minReadySeconds int) error {
@@ -2390,22 +2445,6 @@ func waitForPartialEvents(c *client.Client, ns string, objOrRef runtime.Object,
 	})
 }
 
-// waitForRollbackDone waits for the given deployment finishes rollback.
-func waitForRollbackDone(c *clientset.Clientset, deployment *extensions.Deployment) (err error) {
-	deployments := c.Extensions().Deployments(deployment.Namespace)
-	name := deployment.Name
-	return wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
-		if deployment, err = deployments.Get(name); err != nil {
-			return false, err
-		}
-		// When deployment's RollbackTo is empty, the rollback is done.
-		if deployment.Spec.RollbackTo == nil {
-			return true, nil
-		}
-		return false, nil
-	})
-}
-
 type updateDeploymentFunc func(d *extensions.Deployment)
 
 func updateDeploymentWithRetries(c *clientset.Clientset, namespace, name string, applyUpdate updateDeploymentFunc) (deployment *extensions.Deployment, err error) {