revamp deployment upgrade test

test/e2e/framework/deployment_util.go

@@ -289,3 +289,22 @@ func RunDeployment(config testutils.DeploymentConfig) error {
 func logPodsOfDeployment(c clientset.Interface, deployment *extensions.Deployment, rsList []*extensions.ReplicaSet) {
 	testutils.LogPodsOfDeployment(c, deployment, rsList, Logf)
 }
+
+func WaitForDeploymentCompletes(c clientset.Interface, deployment *extensions.Deployment) error {
+	return testutils.WaitForDeploymentCompletes(c, deployment, Logf, Poll, pollLongTimeout)
+}
+
+func WaitForDeploymentRevision(c clientset.Interface, d *extensions.Deployment, targetRevision string) error {
+	err := wait.PollImmediate(Poll, pollLongTimeout, func() (bool, error) {
+		deployment, err := c.ExtensionsV1beta1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		revision := deployment.Annotations[deploymentutil.RevisionAnnotation]
+		return revision == targetRevision, nil
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for revision to become %q for deployment %q: %v", targetRevision, d.Name, err)
+	}
+	return nil
+}

test/e2e/upgrades/apps/deployments.go

@@ -21,75 +21,63 @@ import (
 
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
-	"k8s.io/kubernetes/pkg/util/version"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/upgrades"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
 
+const (
+	deploymentName = "dp"
+)
+
 // TODO: Test that the deployment stays available during master (and maybe
 // node and cluster upgrades).
 
 // DeploymentUpgradeTest tests that a deployment is using the same replica
 // sets before and after a cluster upgrade.
 type DeploymentUpgradeTest struct {
-	oldD     *extensions.Deployment
+	oldDeploymentUID types.UID
-	updatedD *extensions.Deployment
+	oldRSUID         types.UID
-	oldRS    *extensions.ReplicaSet
+	newRSUID         types.UID
-	newRS    *extensions.ReplicaSet
 }
 
 func (DeploymentUpgradeTest) Name() string { return "[sig-apps] deployment-upgrade" }
 
-func (DeploymentUpgradeTest) Skip(upgCtx upgrades.UpgradeContext) bool {
+// Setup creates a deployment and makes sure it has a new and an old replicaset running.
-	// The Deployment upgrade test currently relies on implementation details to probe the
-	// ReplicaSets belonging to a Deployment. As of 1.7, the client code we call into no
-	// longer supports talking to a server <1.6. (see #47685)
-	minVersion := version.MustParseSemantic("v1.6.0")
-
-	for _, vCtx := range upgCtx.Versions {
-		if vCtx.Version.LessThan(minVersion) {
-			return true
-		}
-	}
-	return false
-}
-
-var _ upgrades.Skippable = DeploymentUpgradeTest{}
-
-// Setup creates a deployment and makes sure it has a new and an old replica set running.
-// This calls in to client code and should not be expected to work against a cluster more than one minor version away from the current version.
 func (t *DeploymentUpgradeTest) Setup(f *framework.Framework) {
-	deploymentName := "deployment-hash-test"
 	c := f.ClientSet
 	nginxImage := imageutils.GetE2EImage(imageutils.NginxSlim)
 
 	ns := f.Namespace.Name
+	deploymentClient := c.ExtensionsV1beta1().Deployments(ns)
+	rsClient := c.ExtensionsV1beta1().ReplicaSets(ns)
 
-	By(fmt.Sprintf("Creating a deployment %q in namespace %q", deploymentName, ns))
+	By(fmt.Sprintf("Creating a deployment %q with 1 replica in namespace %q", deploymentName, ns))
 	d := framework.NewDeployment(deploymentName, int32(1), map[string]string{"test": "upgrade"}, "nginx", nginxImage, extensions.RollingUpdateDeploymentStrategyType)
-	deployment, err := c.Extensions().Deployments(ns).Create(d)
+	deployment, err := deploymentClient.Create(d)
-	framework.ExpectNoError(err)
-
-	// Wait for it to be updated to revision 1
-	By(fmt.Sprintf("Waiting deployment %q to be updated to revision 1", deploymentName))
-	err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", nginxImage)
 	framework.ExpectNoError(err)
 
 	By(fmt.Sprintf("Waiting deployment %q to complete", deploymentName))
-	framework.ExpectNoError(framework.WaitForDeploymentStatusValid(c, deployment))
+	framework.ExpectNoError(framework.WaitForDeploymentCompletes(c, deployment))
 
-	rs, err := deploymentutil.GetNewReplicaSet(deployment, c.ExtensionsV1beta1())
+	By(fmt.Sprintf("Getting replicaset revision 1 of deployment %q", deploymentName))
+	rsSelector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector)
 	framework.ExpectNoError(err)
-	if rs == nil {
+	rsList, err := rsClient.List(metav1.ListOptions{LabelSelector: rsSelector.String()})
-		framework.ExpectNoError(fmt.Errorf("expected a new replica set for deployment %q, found none", deployment.Name))
+	framework.ExpectNoError(err)
+	rss := rsList.Items
+	if len(rss) != 1 {
+		framework.ExpectNoError(fmt.Errorf("expected one replicaset, got %d", len(rss)))
 	}
+	t.oldRSUID = rss[0].UID
 
-	// Store the old replica set - should be the same after the upgrade.
+	By(fmt.Sprintf("Waiting for revision of the deployment %q to become 1", deploymentName))
-	t.oldRS = rs
+	framework.ExpectNoError(framework.WaitForDeploymentRevision(c, deployment, "1"))
 
 	// Trigger a new rollout so that we have some history.
 	By(fmt.Sprintf("Triggering a new rollout for deployment %q", deploymentName))
@@ -98,89 +86,87 @@ func (t *DeploymentUpgradeTest) Setup(f *framework.Framework) {
 	})
 	framework.ExpectNoError(err)
 
-	// Use observedGeneration to determine if the controller noticed the pod template update.
-	framework.Logf("Wait deployment %q to be observed by the deployment controller", deploymentName)
-	framework.ExpectNoError(framework.WaitForObservedDeployment(c, ns, deploymentName, deployment.Generation))
-
-	// Wait for it to be updated to revision 2
-	By(fmt.Sprintf("Waiting deployment %q to be updated to revision 2", deploymentName))
-	framework.ExpectNoError(framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "2", nginxImage))
-
 	By(fmt.Sprintf("Waiting deployment %q to complete", deploymentName))
-	framework.ExpectNoError(framework.WaitForDeploymentStatus(c, deployment))
+	framework.ExpectNoError(framework.WaitForDeploymentCompletes(c, deployment))
 
-	rs, err = deploymentutil.GetNewReplicaSet(deployment, c.ExtensionsV1beta1())
+	By(fmt.Sprintf("Getting replicasets revision 1 and 2 of deployment %q", deploymentName))
+	rsList, err = rsClient.List(metav1.ListOptions{LabelSelector: rsSelector.String()})
 	framework.ExpectNoError(err)
-	if rs == nil {
+	rss = rsList.Items
-		framework.ExpectNoError(fmt.Errorf("expected a new replica set for deployment %q", deployment.Name))
+	if len(rss) != 2 {
+		framework.ExpectNoError(fmt.Errorf("expected 2 replicaset, got %d", len(rss)))
 	}
 
-	if rs.UID == t.oldRS.UID {
+	By(fmt.Sprintf("Checking replicaset of deployment %q that is created before rollout survives the rollout", deploymentName))
-		framework.ExpectNoError(fmt.Errorf("expected a new replica set different from the previous one"))
+	switch t.oldRSUID {
+	case rss[0].UID:
+		t.newRSUID = rss[1].UID
+	case rss[1].UID:
+		t.newRSUID = rss[0].UID
+	default:
+		framework.ExpectNoError(fmt.Errorf("old replicaset with UID %q does not survive rollout", t.oldRSUID))
 	}
 
-	// Store new replica set - should be the same after the upgrade.
+	By(fmt.Sprintf("Waiting for revision of the deployment %q to become 2", deploymentName))
-	t.newRS = rs
+	framework.ExpectNoError(framework.WaitForDeploymentRevision(c, deployment, "2"))
-	deployment, err = c.Extensions().Deployments(ns).Get(deployment.Name, metav1.GetOptions{})
+
-	framework.ExpectNoError(err)
+	t.oldDeploymentUID = deployment.UID
-	t.oldD = deployment
 }
 
-// Test checks whether the replica sets for a deployment are the same after an upgrade.
+// Test checks whether the replicasets for a deployment are the same after an upgrade.
 func (t *DeploymentUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
 	// Block until upgrade is done
-	By(fmt.Sprintf("Waiting for upgrade to finish before checking replica sets for deployment %q", t.oldD.Name))
+	By(fmt.Sprintf("Waiting for upgrade to finish before checking replicasets for deployment %q", deploymentName))
 	<-done
 
 	c := f.ClientSet
+	ns := f.Namespace.Name
+	deploymentClient := c.ExtensionsV1beta1().Deployments(ns)
+	rsClient := c.ExtensionsV1beta1().ReplicaSets(ns)
 
-	deployment, err := c.Extensions().Deployments(t.oldD.Namespace).Get(t.oldD.Name, metav1.GetOptions{})
+	deployment, err := deploymentClient.Get(deploymentName, metav1.GetOptions{})
 	framework.ExpectNoError(err)
-	t.updatedD = deployment
 
-	By(fmt.Sprintf("Waiting for deployment %q to complete adoption", deployment.Name))
+	By(fmt.Sprintf("Checking UID to verify deployment %q survives upgrade", deploymentName))
-	framework.ExpectNoError(framework.WaitForDeploymentStatus(c, deployment))
+	Expect(deployment.UID).To(Equal(t.oldDeploymentUID))
 
-	By(fmt.Sprintf("Checking that replica sets for deployment %q are the same as prior to the upgrade", t.updatedD.Name))
+	By(fmt.Sprintf("Verifying deployment %q does not create new replicasets", deploymentName))
-	_, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(t.updatedD, c.ExtensionsV1beta1())
+	rsSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
 	framework.ExpectNoError(err)
-	if newRS == nil {
+	rsList, err := rsClient.List(metav1.ListOptions{LabelSelector: rsSelector.String()})
-		By(t.spewDeploymentAndReplicaSets(newRS, allOldRSs))
+	framework.ExpectNoError(err)
-		framework.ExpectNoError(fmt.Errorf("expected a new replica set for deployment %q", t.updatedD.Name))
+	rss := rsList.Items
+	if len(rss) != 2 {
+		framework.ExpectNoError(fmt.Errorf("expected 2 replicaset, got %d", len(rss)))
 	}
-	if newRS.UID != t.newRS.UID {
+
-		By(t.spewDeploymentAndReplicaSets(newRS, allOldRSs))
+	switch t.oldRSUID {
-		framework.ExpectNoError(fmt.Errorf("expected new replica set:\n%#v\ngot new replica set:\n%#v\n", t.newRS, newRS))
+	case rss[0].UID:
-	}
+		Expect(rss[1].UID).To(Equal(t.newRSUID))
-	if len(allOldRSs) != 1 {
+	case rss[1].UID:
-		By(t.spewDeploymentAndReplicaSets(newRS, allOldRSs))
+		Expect(rss[0].UID).To(Equal(t.newRSUID))
-		errString := fmt.Sprintf("expected one old replica set, got %d\n", len(allOldRSs))
+	default:
-		for i := range allOldRSs {
+		framework.ExpectNoError(fmt.Errorf("new replicasets are created during upgrade of deployment %q", deploymentName))
-			rs := allOldRSs[i]
-			errString += fmt.Sprintf("%#v\n", rs)
-		}
-		framework.ExpectNoError(fmt.Errorf(errString))
-	}
-	if allOldRSs[0].UID != t.oldRS.UID {
-		By(t.spewDeploymentAndReplicaSets(newRS, allOldRSs))
-		framework.ExpectNoError(fmt.Errorf("expected old replica set:\n%#v\ngot old replica set:\n%#v\n", t.oldRS, allOldRSs[0]))
 	}
+
+	By(fmt.Sprintf("Verifying revision of the deployment %q is still 2", deploymentName))
+	Expect(deployment.Annotations[deploymentutil.RevisionAnnotation]).To(Equal("2"))
+
+	By(fmt.Sprintf("Waiting for deployment %q to complete adoption", deploymentName))
+	framework.ExpectNoError(framework.WaitForDeploymentCompletes(c, deployment))
+
+	// Verify the upgraded deployment is active by scaling up the deployment by 1
+	By(fmt.Sprintf("Scaling up replicaset of deployment %q by 1", deploymentName))
+	_, err = framework.UpdateDeploymentWithRetries(c, ns, deploymentName, func(deployment *extensions.Deployment) {
+		*deployment.Spec.Replicas = *deployment.Spec.Replicas + 1
+	})
+	framework.ExpectNoError(err)
+
+	By(fmt.Sprintf("Waiting for deployment %q to complete after scaling", deploymentName))
+	framework.ExpectNoError(framework.WaitForDeploymentCompletes(c, deployment))
 }
 
 // Teardown cleans up any remaining resources.
 func (t *DeploymentUpgradeTest) Teardown(f *framework.Framework) {
 	// rely on the namespace deletion to clean up everything
 }
-
-func (t *DeploymentUpgradeTest) spewDeploymentAndReplicaSets(newRS *extensions.ReplicaSet, allOldRSs []*extensions.ReplicaSet) string {
-	msg := fmt.Sprintf("deployment prior to the upgrade:\n%#v\n", t.oldD)
-	msg += fmt.Sprintf("old replica sets prior to the upgrade:\n%#v\n", t.oldRS)
-	msg += fmt.Sprintf("new replica sets prior to the upgrade:\n%#v\n", t.newRS)
-	msg += fmt.Sprintf("deployment after the upgrade:\n%#v\n", t.updatedD)
-	msg += fmt.Sprintf("new replica set after the upgrade:\n%#v\n", newRS)
-	msg += fmt.Sprintf("old replica sets after the upgrade:\n")
-	for i := range allOldRSs {
-		msg += fmt.Sprintf("%#v\n", allOldRSs[i])
-	}
-	return msg
-}

test/utils/deployment.go

@@ -244,3 +244,26 @@ func WaitForObservedDeployment(c clientset.Interface, ns, deploymentName string,
 		return c.Extensions().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
 	}, desiredGeneration, 2*time.Second, 1*time.Minute)
 }
+
+// Pool until deployment status and its underlying resources reach the desired state.
+func WaitForDeploymentCompletes(c clientset.Interface, d *extensions.Deployment, logf LogfFn, pollInterval, pollTimeout time.Duration) error {
+	var reason string
+	err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
+		deployment, err := c.ExtensionsV1beta1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		// When the deployment status and its underlying resources reach the desired state, we're done
+		if deploymentutil.DeploymentComplete(d, &deployment.Status) {
+			return true, nil
+		}
+		reason = fmt.Sprintf("deployment status: %#v", deployment.Status)
+		logf(reason)
+		return false, nil
+	})
+
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("timeout waiting for deployment to complete: %v, most recent deployment status: %s", err, reason)
+	}
+	return err
+}