Update to include watch tooling

test/e2e/apps/rc.go

@@ -32,9 +32,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
-	watch "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+	watch "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/kubernetes/pkg/controller/replication"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -104,6 +105,14 @@ var _ = SIGDescribe("ReplicationController", func() {
 		testRcInitialReplicaCount := int32(1)
 		testRcMaxReplicaCount := int32(2)
 		rcResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "replicationcontrollers"}
+		expectedWatchEvents := []watch.Event{
+			{Type: watch.Added},
+			{Type: watch.Modified},
+			{Type: watch.Modified},
+			{Type: watch.Modified},
+			{Type: watch.Modified},
+			{Type: watch.Deleted},
+		}
 
 		rcTest := v1.ReplicationController{
 			ObjectMeta: metav1.ObjectMeta{
@@ -128,39 +137,49 @@ var _ = SIGDescribe("ReplicationController", func() {
 			},
 		}
 
+		framework.WatchEventSequenceVerifier(context.TODO(), dc, rcResource, testRcNamespace, testRcName, metav1.ListOptions{LabelSelector: "test-rc-static=true"}, expectedWatchEvents, func(retryWatcher *watchtools.RetryWatcher) (actualWatchEvents []watch.Event) {
 			ginkgo.By("creating a ReplicationController")
 			// Create a ReplicationController
 			_, err := f.ClientSet.CoreV1().ReplicationControllers(testRcNamespace).Create(context.TODO(), &rcTest, metav1.CreateOptions{})
 			framework.ExpectNoError(err, "Failed to create ReplicationController")
 
-		// setup a watch for the RC
-		rcWatchTimeoutSeconds := int64(180)
-		rcWatch, err := f.ClientSet.CoreV1().ReplicationControllers(testRcNamespace).Watch(context.TODO(), metav1.ListOptions{LabelSelector: "test-rc-static=true", TimeoutSeconds: &rcWatchTimeoutSeconds})
-		framework.ExpectNoError(err, "Failed to setup watch on newly created ReplicationController")
-
-		rcWatchChan := rcWatch.ResultChan()
-
 			ginkgo.By("waiting for RC to be added")
 			eventFound := false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Added {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Added {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
 			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
 
 			ginkgo.By("waiting for available Replicas")
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			rc, ok := watchEvent.Object.(*v1.ReplicationController)
+			defer cancel()
-			framework.ExpectEqual(ok, true, "Unable to convert type of ReplicationController watch watchEvent")
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
-			if rc.Status.Replicas == testRcInitialReplicaCount && rc.Status.ReadyReplicas == testRcInitialReplicaCount {
+				var rc *v1.ReplicationController
+				rcBytes, err := json.Marshal(watchEvent.Object)
+				if err != nil {
+					return false, err
+				}
+				err = json.Unmarshal(rcBytes, &rc)
+				if err != nil {
+					return false, err
+				}
+				if rc.Status.Replicas != testRcInitialReplicaCount || rc.Status.ReadyReplicas != testRcInitialReplicaCount {
+					return false, nil
+				}
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait for condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find event with initial replica count")
+			framework.ExpectEqual(eventFound, true, "RC has not reached ReadyReplicas count of %v", testRcInitialReplicaCount)
 
 			rcLabelPatchPayload, err := json.Marshal(v1.ReplicationController{
 				ObjectMeta: metav1.ObjectMeta{
@@ -175,13 +194,18 @@ var _ = SIGDescribe("ReplicationController", func() {
 			framework.ExpectEqual(testRcPatched.ObjectMeta.Labels["test-rc"], "patched", "failed to patch RC")
 			ginkgo.By("waiting for RC to be modified")
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Modified {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Modified {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Modified)
+			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
 
 			rcStatusPatchPayload, err := json.Marshal(map[string]interface{}{
 				"status": map[string]interface{}{
@@ -198,25 +222,36 @@ var _ = SIGDescribe("ReplicationController", func() {
 			framework.ExpectEqual(rcStatus.Status.ReadyReplicas, int32(0), "ReplicationControllerStatus's readyReplicas does not equal 0")
 			ginkgo.By("waiting for RC to be modified")
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Modified {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Modified {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Modified)
+			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
 
-		eventFound = false
+			ginkgo.By("waiting for available Replicas")
-		for watchEvent := range rcWatchChan {
+			_, err = framework.WatchUntilWithoutRetry(context.TODO(), retryWatcher, func(watchEvent watch.Event) (bool, error) {
-			rc, ok := watchEvent.Object.(*v1.ReplicationController)
+				var rc *v1.ReplicationController
-			framework.ExpectEqual(ok, true, "Unable to convert type of ReplicationController watch watchEvent")
+				rcBytes, err := json.Marshal(watchEvent.Object)
-
+				if err != nil {
-			if rc.Status.Replicas == testRcInitialReplicaCount {
+					return false, err
-				eventFound = true
-				break
 				}
+				err = json.Unmarshal(rcBytes, &rc)
+				if err != nil {
+					return false, err
 				}
-		framework.ExpectEqual(eventFound, true, "RC does not have initial replica count")
+				if rc.Status.Replicas != testRcInitialReplicaCount {
+					return false, nil
+				}
+				return true, nil
+			})
+			framework.ExpectEqual(eventFound, true, "Failed to find updated ready replica count")
 
 			ginkgo.By("fetching ReplicationController status")
 			rcStatusUnstructured, err := dc.Resource(rcResource).Namespace(testRcNamespace).Get(context.TODO(), testRcName, metav1.GetOptions{}, "status")
@@ -239,30 +274,40 @@ var _ = SIGDescribe("ReplicationController", func() {
 			_, err = f.ClientSet.CoreV1().ReplicationControllers(testRcNamespace).Patch(context.TODO(), testRcName, types.StrategicMergePatchType, []byte(rcScalePatchPayload), metav1.PatchOptions{}, "scale")
 			framework.ExpectNoError(err, "Failed to patch ReplicationControllerScale")
 			ginkgo.By("waiting for RC to be modified")
-
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Modified {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Modified {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Modified)
+			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
 
-		var rcFromWatch *v1.ReplicationController
 			ginkgo.By("waiting for ReplicationController's scale to be the max amount")
-		foundRcWithMaxScale := false
+			eventFound = false
-		for watchEvent := range rcWatchChan {
+			_, err = framework.WatchUntilWithoutRetry(context.TODO(), retryWatcher, func(watchEvent watch.Event) (bool, error) {
-			rc, ok := watchEvent.Object.(*v1.ReplicationController)
+				var rc *v1.ReplicationController
-			framework.ExpectEqual(ok, true, "Unable to convert type of ReplicationController watch watchEvent")
+				rcBytes, err := json.Marshal(watchEvent.Object)
-			if rc.ObjectMeta.Name == testRcName && rc.ObjectMeta.Namespace == testRcNamespace && rc.Status.Replicas == testRcMaxReplicaCount && rc.Status.ReadyReplicas == testRcMaxReplicaCount {
+				if err != nil {
-				foundRcWithMaxScale = true
+					return false, err
-				rcFromWatch = rc
-				break
 				}
+				err = json.Unmarshal(rcBytes, &rc)
+				if err != nil {
+					return false, err
 				}
-		framework.ExpectEqual(foundRcWithMaxScale, true, "failed to locate a ReplicationController with max scale")
+				if rc.ObjectMeta.Name != testRcName || rc.ObjectMeta.Namespace != testRcNamespace || rc.Status.Replicas != testRcMaxReplicaCount || rc.Status.ReadyReplicas != testRcMaxReplicaCount {
-		framework.ExpectEqual(rcFromWatch.Status.Replicas, testRcMaxReplicaCount, "ReplicationController ReplicasSet Scale does not match the expected scale")
+					return false, nil
+				}
+				eventFound = true
+				return true, nil
+			})
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
+			framework.ExpectEqual(eventFound, true, "Failed to find updated ready replica count")
 
 			// Get the ReplicationController
 			ginkgo.By("fetching ReplicationController; ensuring that it's patched")
@@ -276,19 +321,23 @@ var _ = SIGDescribe("ReplicationController", func() {
 
 			// Replace the ReplicationController's status
 			ginkgo.By("updating ReplicationController status")
-		rcStatus, err = f.ClientSet.CoreV1().ReplicationControllers(testRcNamespace).UpdateStatus(context.TODO(), rcStatusUpdatePayload, metav1.UpdateOptions{})
+			_, err = f.ClientSet.CoreV1().ReplicationControllers(testRcNamespace).UpdateStatus(context.TODO(), rcStatusUpdatePayload, metav1.UpdateOptions{})
 			framework.ExpectNoError(err, "failed to update ReplicationControllerStatus")
-		framework.ExpectEqual(rcStatus.Status.ReadyReplicas, int32(1), "ReplicationControllerStatus readyReplicas does not equal 1")
-		ginkgo.By("waiting for RC to be modified")
 
+			ginkgo.By("waiting for RC to be modified")
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Modified {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Modified {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Modified)
+			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
 
 			ginkgo.By("listing all ReplicationControllers")
 			rcs, err := f.ClientSet.CoreV1().ReplicationControllers("").List(context.TODO(), metav1.ListOptions{LabelSelector: "test-rc-static=true"})
@@ -314,13 +363,21 @@ var _ = SIGDescribe("ReplicationController", func() {
 
 			ginkgo.By("waiting for ReplicationController to have a DELETED watchEvent")
 			eventFound = false
-		for watchEvent := range rcWatchChan {
+			ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
-			if watchEvent.Type == watch.Deleted {
+			defer cancel()
+			_, err = framework.WatchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) {
+				if watchEvent.Type != watch.Deleted {
+					return false, nil
+				}
+				actualWatchEvents = append(actualWatchEvents, watchEvent)
 				eventFound = true
-				break
+				return true, nil
-			}
+			})
-		}
+			framework.ExpectNoError(err, "Wait until condition with watch events should not return an error")
-		framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Deleted)
+			framework.ExpectEqual(eventFound, true, "failed to find RC %v event", watch.Added)
+
+			return actualWatchEvents
+		})
 	})
 })