Move deployment e2e test for rollback with no revision to integration

2025-09-15 14:14:39 +00:00 · 2017-10-13 14:09:46 -07:00
parent 4de496d4f8
commit 1ca6120af3
8 changed files with 369 additions and 342 deletions
--- a/test/utils/deployment.go
+++ b/test/utils/deployment.go
@@ -68,58 +68,34 @@ func LogPodsOfDeployment(c clientset.Interface, deployment *extensions.Deploymen
 	}
 }

-// Waits for the deployment status to become valid (i.e. max unavailable and max surge aren't violated anymore).
-// Note that the status should stay valid at all times unless shortly after a scaling event or the deployment is just created.
-// To verify that the deployment status is valid and wait for the rollout to finish, use WaitForDeploymentStatus instead.
-func WaitForDeploymentStatusValid(c clientset.Interface, d *extensions.Deployment, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
+// Waits for the deployment to complete.
+// If during a rolling update (rolling == true), returns an error if the deployment's
+// rolling update strategy (max unavailable or max surge) is broken at any times.
+// It's not seen as a rolling update if shortly after a scaling event or the deployment is just created.
+func waitForDeploymentCompleteMaybeCheckRolling(c clientset.Interface, d *extensions.Deployment, rolling bool, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
 	var (
-		oldRSs, allOldRSs, allRSs []*extensions.ReplicaSet
-		newRS                     *extensions.ReplicaSet
-		deployment                *extensions.Deployment
-		reason                    string
+		deployment *extensions.Deployment
+		reason     string
 	)

-	err := wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
+	err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
 		var err error
-		deployment, err = c.Extensions().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
+		deployment, err = c.ExtensionsV1beta1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
-		oldRSs, allOldRSs, newRS, err = deploymentutil.GetAllReplicaSets(deployment, c.ExtensionsV1beta1())
-		if err != nil {
-			return false, err
-		}
-		if newRS == nil {
-			// New RC hasn't been created yet.
-			reason = "new replica set hasn't been created yet"
-			logf(reason)
-			return false, nil
-		}
-		allRSs = append(oldRSs, newRS)
-		// The old/new ReplicaSets need to contain the pod-template-hash label
-		for i := range allRSs {
-			if !labelsutil.SelectorHasLabel(allRSs[i].Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) {
-				reason = "all replica sets need to contain the pod-template-hash label"
-				logf(reason)
-				return false, nil
+
+		// If during a rolling update, make sure rolling update strategy isn't broken at any times.
+		if rolling {
+			reason, err = checkRollingUpdateStatus(c, deployment, logf)
+			if err != nil {
+				return false, err
 			}
-		}
-		totalCreated := deploymentutil.GetReplicaCountForReplicaSets(allRSs)
-		maxCreated := *(deployment.Spec.Replicas) + deploymentutil.MaxSurge(*deployment)
-		if totalCreated > maxCreated {
-			reason = fmt.Sprintf("total pods created: %d, more than the max allowed: %d", totalCreated, maxCreated)
 			logf(reason)
-			return false, nil
-		}
-		minAvailable := deploymentutil.MinAvailable(deployment)
-		if deployment.Status.AvailableReplicas < minAvailable {
-			reason = fmt.Sprintf("total pods available: %d, less than the min required: %d", deployment.Status.AvailableReplicas, minAvailable)
-			logf(reason)
-			return false, nil
 		}

 		// When the deployment status and its underlying resources reach the desired state, we're done
-		if deploymentutil.DeploymentComplete(deployment, &deployment.Status) {
+		if deploymentutil.DeploymentComplete(d, &deployment.Status) {
 			return true, nil
 		}

@@ -130,8 +106,6 @@ func WaitForDeploymentStatusValid(c clientset.Interface, d *extensions.Deploymen
 	})

 	if err == wait.ErrWaitTimeout {
-		LogReplicaSetsOfDeployment(deployment, allOldRSs, newRS, logf)
-		LogPodsOfDeployment(c, deployment, allRSs, logf)
 		err = fmt.Errorf("%s", reason)
 	}
 	if err != nil {
@@ -140,6 +114,58 @@ func WaitForDeploymentStatusValid(c clientset.Interface, d *extensions.Deploymen
 	return nil
 }

+func checkRollingUpdateStatus(c clientset.Interface, deployment *extensions.Deployment, logf LogfFn) (string, error) {
+	var reason string
+	oldRSs, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(deployment, c.ExtensionsV1beta1())
+	if err != nil {
+		return "", err
+	}
+	if newRS == nil {
+		// New RC hasn't been created yet.
+		reason = "new replica set hasn't been created yet"
+		return reason, nil
+	}
+	allRSs := append(oldRSs, newRS)
+	// The old/new ReplicaSets need to contain the pod-template-hash label
+	for i := range allRSs {
+		if !labelsutil.SelectorHasLabel(allRSs[i].Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) {
+			reason = "all replica sets need to contain the pod-template-hash label"
+			return reason, nil
+		}
+	}
+
+	// Check max surge and min available
+	totalCreated := deploymentutil.GetReplicaCountForReplicaSets(allRSs)
+	maxCreated := *(deployment.Spec.Replicas) + deploymentutil.MaxSurge(*deployment)
+	if totalCreated > maxCreated {
+		LogReplicaSetsOfDeployment(deployment, allOldRSs, newRS, logf)
+		LogPodsOfDeployment(c, deployment, allRSs, logf)
+		return "", fmt.Errorf("total pods created: %d, more than the max allowed: %d", totalCreated, maxCreated)
+	}
+	minAvailable := deploymentutil.MinAvailable(deployment)
+	if deployment.Status.AvailableReplicas < minAvailable {
+		LogReplicaSetsOfDeployment(deployment, allOldRSs, newRS, logf)
+		LogPodsOfDeployment(c, deployment, allRSs, logf)
+		return "", fmt.Errorf("total pods available: %d, less than the min required: %d", deployment.Status.AvailableReplicas, minAvailable)
+	}
+	return "", nil
+}
+
+// Waits for the deployment to complete, and check rolling update strategy isn't broken at any times.
+// Rolling update strategy should not be broken during a rolling update.
+func WaitForDeploymentCompleteAndCheckRolling(c clientset.Interface, d *extensions.Deployment, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
+	rolling := true
+	return waitForDeploymentCompleteMaybeCheckRolling(c, d, rolling, logf, pollInterval, pollTimeout)
+}
+
+// Waits for the deployment to complete, and don't check if rolling update strategy is broken.
+// Rolling update strategy is used only during a rolling update, and can be violated in other situations,
+// such as shortly after a scaling event or the deployment is just created.
+func WaitForDeploymentComplete(c clientset.Interface, d *extensions.Deployment, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
+	rolling := false
+	return waitForDeploymentCompleteMaybeCheckRolling(c, d, rolling, logf, pollInterval, pollTimeout)
+}
+
 // WaitForDeploymentRevisionAndImage waits for the deployment's and its new RS's revision and container image to match the given revision and image.
 // Note that deployment revision and its new RS revision should be updated shortly, so we only wait for 1 minute here to fail early.
 func WaitForDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName string, revision, image string, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
@@ -148,45 +174,17 @@ func WaitForDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName
 	var reason string
 	err := wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
 		var err error
-		deployment, err = c.Extensions().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
+		deployment, err = c.ExtensionsV1beta1().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
 		// The new ReplicaSet needs to be non-nil and contain the pod-template-hash label
-
 		newRS, err = deploymentutil.GetNewReplicaSet(deployment, c.ExtensionsV1beta1())
-
 		if err != nil {
 			return false, err
 		}
-		if newRS == nil {
-			reason = fmt.Sprintf("New replica set for deployment %q is yet to be created", deployment.Name)
-			logf(reason)
-			return false, nil
-		}
-		if !labelsutil.SelectorHasLabel(newRS.Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) {
-			reason = fmt.Sprintf("New replica set %q doesn't have DefaultDeploymentUniqueLabelKey", newRS.Name)
-			logf(reason)
-			return false, nil
-		}
-		// Check revision of this deployment, and of the new replica set of this deployment
-		if deployment.Annotations == nil || deployment.Annotations[deploymentutil.RevisionAnnotation] != revision {
-			reason = fmt.Sprintf("Deployment %q doesn't have the required revision set", deployment.Name)
-			logf(reason)
-			return false, nil
-		}
-		if !containsImage(deployment.Spec.Template.Spec.Containers, image) {
-			reason = fmt.Sprintf("Deployment %q doesn't have the required image %s set", deployment.Name, image)
-			logf(reason)
-			return false, nil
-		}
-		if newRS.Annotations == nil || newRS.Annotations[deploymentutil.RevisionAnnotation] != revision {
-			reason = fmt.Sprintf("New replica set %q doesn't have the required revision set", newRS.Name)
-			logf(reason)
-			return false, nil
-		}
-		if !containsImage(newRS.Spec.Template.Spec.Containers, image) {
-			reason = fmt.Sprintf("New replica set %q doesn't have the required image %s.", newRS.Name, image)
+		if err := checkRevisionAndImage(deployment, newRS, revision, image); err != nil {
+			reason = err.Error()
 			logf(reason)
 			return false, nil
 		}
@@ -205,6 +203,46 @@ func WaitForDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName
 	return nil
 }

+// CheckDeploymentRevisionAndImage checks if the input deployment's and its new replica set's revision and image are as expected.
+func CheckDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName, revision, image string) error {
+	deployment, err := c.ExtensionsV1beta1().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to get deployment %s during revision check: %v", deploymentName, err)
+	}
+
+	// Check revision of the new replica set of this deployment
+	newRS, err := deploymentutil.GetNewReplicaSet(deployment, c.ExtensionsV1beta1())
+	if err != nil {
+		return fmt.Errorf("unable to get new replicaset of deployment %s during revision check: %v", deploymentName, err)
+	}
+	return checkRevisionAndImage(deployment, newRS, revision, image)
+}
+
+func checkRevisionAndImage(deployment *extensions.Deployment, newRS *extensions.ReplicaSet, revision, image string) error {
+	// The new ReplicaSet needs to be non-nil and contain the pod-template-hash label
+	if newRS == nil {
+		return fmt.Errorf("new replicaset for deployment %q is yet to be created", deployment.Name)
+	}
+	if !labelsutil.SelectorHasLabel(newRS.Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) {
+		return fmt.Errorf("new replica set %q doesn't have %q label selector", newRS.Name, extensions.DefaultDeploymentUniqueLabelKey)
+	}
+	// Check revision of this deployment, and of the new replica set of this deployment
+	if deployment.Annotations == nil || deployment.Annotations[deploymentutil.RevisionAnnotation] != revision {
+		return fmt.Errorf("deployment %q doesn't have the required revision set", deployment.Name)
+	}
+	if newRS.Annotations == nil || newRS.Annotations[deploymentutil.RevisionAnnotation] != revision {
+		return fmt.Errorf("new replicaset %q doesn't have the required revision set", newRS.Name)
+	}
+	// Check the image of this deployment, and of the new replica set of this deployment
+	if !containsImage(deployment.Spec.Template.Spec.Containers, image) {
+		return fmt.Errorf("deployment %q doesn't have the required image %s set", deployment.Name, image)
+	}
+	if !containsImage(newRS.Spec.Template.Spec.Containers, image) {
+		return fmt.Errorf("new replica set %q doesn't have the required image %s.", newRS.Name, image)
+	}
+	return nil
+}
+
 func containsImage(containers []v1.Container, imageName string) bool {
 	for _, container := range containers {
 		if container.Image == imageName {
@@ -245,25 +283,21 @@ func WaitForObservedDeployment(c clientset.Interface, ns, deploymentName string,
 	}, desiredGeneration, 2*time.Second, 1*time.Minute)
 }

-// Pool until deployment status and its underlying resources reach the desired state.
-func WaitForDeploymentCompletes(c clientset.Interface, d *extensions.Deployment, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
-	var reason string
+// WaitForDeploymentRollbackCleared waits for given deployment either started rolling back or doesn't need to rollback.
+func WaitForDeploymentRollbackCleared(c clientset.Interface, ns, deploymentName string, pollInterval, pollTimeout time.Duration) error {
 	err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
-		deployment, err := c.ExtensionsV1beta1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
+		deployment, err := c.ExtensionsV1beta1().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
-		// When the deployment status and its underlying resources reach the desired state, we're done
-		if deploymentutil.DeploymentComplete(d, &deployment.Status) {
+		// Rollback not set or is kicked off
+		if deployment.Spec.RollbackTo == nil {
 			return true, nil
 		}
-		reason = fmt.Sprintf("deployment status: %#v", deployment.Status)
-		logf(reason)
 		return false, nil
 	})
-
-	if err == wait.ErrWaitTimeout {
-		err = fmt.Errorf("timeout waiting for deployment to complete: %v, most recent deployment status: %s", err, reason)
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %s rollbackTo to be cleared: %v", deploymentName, err)
 	}
-	return err
+	return nil
 }