From 93d1040fcdb4e0ee0ba930b1dabb5ee2b62075a8 Mon Sep 17 00:00:00 2001 From: Robert Rati Date: Wed, 25 Mar 2015 15:35:54 -0400 Subject: [PATCH 1/3] Add checking of events after all pods started to verify no failures in density test #6637 --- test/e2e/density.go | 56 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/test/e2e/density.go b/test/e2e/density.go index 8620fd98503..76c4de2c8ae 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -83,7 +83,8 @@ func DeleteRC(c *client.Client, ns, name string) error { } // Launch a Replication Controller and wait for all pods it spawns -// to become running +// to become running. The controller will need to be cleaned up external +// to this method func RunRC(c *client.Client, name string, ns, image string, replicas int) { defer GinkgoRecover() @@ -91,12 +92,6 @@ func RunRC(c *client.Client, name string, ns, image string, replicas int) { current := 0 same := 0 - defer func() { - By("Cleaning up the replication controller") - err := DeleteRC(c, ns, name) - Expect(err).NotTo(HaveOccurred()) - }() - By(fmt.Sprintf("Creating replication controller %s", name)) _, err := c.ReplicationControllers(ns).Create(&api.ReplicationController{ ObjectMeta: api.ObjectMeta{ @@ -138,7 +133,7 @@ func RunRC(c *client.Client, name string, ns, image string, replicas int) { } else if last == current { same++ } else if current < last { - Failf("Controller %s: Number of submitted pods dropped from %d to %d", last, current) + Failf("Controller %s: Number of submitted pods dropped from %d to %d", name, last, current) } if same >= failCount { @@ -228,13 +223,21 @@ var _ = Describe("Density", func() { // during the test so clean it up here rc, err := c.ReplicationControllers(ns).Get(RCName) if err == nil && rc.Spec.Replicas != 0 { - DeleteRC(c, ns, RCName) + By("Cleaning up the replication controller") + err := DeleteRC(c, ns, RCName) + expectNoError(err) + } + + // Clean up the namespace if a non-default one was used + if ns != api.NamespaceDefault { + By("Cleaning up the namespace") + err := c.Namespaces().Delete(ns) + expectNoError(err) } }) // Tests with "Skipped" substring in their name will be skipped when running // e2e test suite without --ginkgo.focus & --ginkgo.skip flags. - type Density struct { skip bool podsPerMinion int @@ -260,9 +263,38 @@ var _ = Describe("Density", func() { } itArg := testArg It(name, func() { + uuid := string(util.NewUUID()) totalPods := itArg.podsPerMinion * minionCount - RCName = "my-hostname-density" + strconv.Itoa(totalPods) + "-" + string(util.NewUUID()) + nameStr := strconv.Itoa(totalPods) + "-" + uuid + ns = "e2e-density" + nameStr + RCName = "my-hostname-density" + nameStr RunRC(c, RCName, ns, "gcr.io/google_containers/pause:go", totalPods) + By("waiting for all events to be recorded") + last := -1 + current := 0 + var events *api.EventList + for last < current { + e, err := c.Events(ns).List( + labels.Everything(), + fields.Set{ + "involvedObject.namespace": ns, + }.AsSelector(), + ) + expectNoError(err) + last = current + current = len(e.Items) + events = e + time.Sleep(10 * time.Second) + } + Logf("Found %d events", current) + + // Verify no pods were killed or failed to start + By("verifying no pods were killed or failed to start") + for _, e := range events.Items { + for _, s := range []string{"kill", "fail"} { + Expect(e.Reason).NotTo(ContainSubstring(s), "event:' %s', reason: '%s', message: '%s', field path: '%s'", e, e.ObjectMeta.Name, e.Message, e.InvolvedObject.FieldPath) + } + } }) } @@ -305,6 +337,8 @@ var _ = Describe("Density", func() { RunRC(c, name, ns, "gcr.io/google_containers/pause:go", n) podsLaunched += n glog.Info("Launched %v pods so far...", podsLaunched) + err := DeleteRC(c, ns, name) + expectNoError(err) } }() } From 23c5b7714e4194be289029145b435c479518ffe2 Mon Sep 17 00:00:00 2001 From: Robert Rati Date: Mon, 20 Apr 2015 12:59:30 -0400 Subject: [PATCH 2/3] Bounded the loop waiting for all events to be logged and print a warning if not all are logged 10 minutes after all pods are started #6637 --- test/e2e/density.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/e2e/density.go b/test/e2e/density.go index 76c4de2c8ae..fc3bc198cd5 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -269,11 +269,13 @@ var _ = Describe("Density", func() { ns = "e2e-density" + nameStr RCName = "my-hostname-density" + nameStr RunRC(c, RCName, ns, "gcr.io/google_containers/pause:go", totalPods) - By("waiting for all events to be recorded") + + By("Waiting for all events to be recorded") last := -1 current := 0 var events *api.EventList - for last < current { + timeout := 10 * time.Minute + for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) { e, err := c.Events(ns).List( labels.Everything(), fields.Set{ @@ -284,12 +286,14 @@ var _ = Describe("Density", func() { last = current current = len(e.Items) events = e - time.Sleep(10 * time.Second) + } + if current != last { + Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes()) } Logf("Found %d events", current) - // Verify no pods were killed or failed to start - By("verifying no pods were killed or failed to start") + // Verify there were no pod killings or failures + By("Verifying there were no pod killings or failures") for _, e := range events.Items { for _, s := range []string{"kill", "fail"} { Expect(e.Reason).NotTo(ContainSubstring(s), "event:' %s', reason: '%s', message: '%s', field path: '%s'", e, e.ObjectMeta.Name, e.Message, e.InvolvedObject.FieldPath) From 020ba6a6c7611e8418d6a7658ca4b1494c0a9ead Mon Sep 17 00:00:00 2001 From: Robert Rati Date: Tue, 21 Apr 2015 13:55:14 -0400 Subject: [PATCH 3/3] Converted from continually listing events to using an Informer #6637 --- test/e2e/density.go | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/e2e/density.go b/test/e2e/density.go index fc3bc198cd5..a380f24bbc6 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -24,10 +24,14 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache" + "github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework" "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait" + "github.com/GoogleCloudPlatform/kubernetes/pkg/watch" "github.com/golang/glog" . "github.com/onsi/ginkgo" @@ -268,25 +272,42 @@ var _ = Describe("Density", func() { nameStr := strconv.Itoa(totalPods) + "-" + uuid ns = "e2e-density" + nameStr RCName = "my-hostname-density" + nameStr + + // Create a listener for events + events := make([](*api.Event), 0) + _, controller := framework.NewInformer( + &cache.ListWatch{ + ListFunc: func() (runtime.Object, error) { + return c.Events(ns).List(labels.Everything(), fields.Everything()) + }, + WatchFunc: func(rv string) (watch.Interface, error) { + return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv) + }, + }, + &api.Event{}, + time.Second*10, + framework.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + events = append(events, obj.(*api.Event)) + }, + }, + ) + stop := make(chan struct{}) + go controller.Run(stop) + + // Start the replication controller RunRC(c, RCName, ns, "gcr.io/google_containers/pause:go", totalPods) By("Waiting for all events to be recorded") last := -1 - current := 0 - var events *api.EventList + current := len(events) timeout := 10 * time.Minute for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) { - e, err := c.Events(ns).List( - labels.Everything(), - fields.Set{ - "involvedObject.namespace": ns, - }.AsSelector(), - ) - expectNoError(err) last = current - current = len(e.Items) - events = e + current = len(events) } + close(stop) + if current != last { Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes()) } @@ -294,7 +315,7 @@ var _ = Describe("Density", func() { // Verify there were no pod killings or failures By("Verifying there were no pod killings or failures") - for _, e := range events.Items { + for _, e := range events { for _, s := range []string{"kill", "fail"} { Expect(e.Reason).NotTo(ContainSubstring(s), "event:' %s', reason: '%s', message: '%s', field path: '%s'", e, e.ObjectMeta.Name, e.Message, e.InvolvedObject.FieldPath) }