From 54f2305decf0f8f754dafd13b6eb62db6638431a Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Jul 2015 13:53:03 +0200 Subject: [PATCH 1/2] Check whether all nodes are healthy in e2e framework --- test/e2e/framework.go | 5 +++++ test/e2e/resize_nodes.go | 4 ++++ test/e2e/util.go | 16 ++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 4f31c914d38..0f7bc590e5b 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -85,6 +85,11 @@ func (f *Framework) afterEach() { // you may or may not see the killing/deletion/cleanup events. } + // Check whether all nodes are ready after the test. + if err := allNodesReady(f.Client); err != nil { + Failf("All nodes should be ready after test, %v", err) + } + By(fmt.Sprintf("Destroying namespace %q for this suite.", f.Namespace.Name)) if err := f.Client.Namespaces().Delete(f.Namespace.Name); err != nil { Failf("Couldn't delete ns %q: %s", f.Namespace.Name, err) diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index d44686d620a..bd3ee2bc26c 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -412,6 +412,10 @@ var _ = Describe("Nodes", func() { }) AfterEach(func() { + By("checking whether all nodes are healthy") + if err := allNodesReady(c); err != nil { + Failf("Not all nodes are ready: %v", err) + } By(fmt.Sprintf("destroying namespace for this suite %s", ns)) if err := c.Namespaces().Delete(ns); err != nil { Failf("Couldn't delete namespace '%s', %v", ns, err) diff --git a/test/e2e/util.go b/test/e2e/util.go index aa160dc04e6..71a66145dfc 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1461,6 +1461,22 @@ func waitForNodeToBe(c *client.Client, name string, wantReady bool, timeout time return false } +// checks whether all registered nodes are ready +func allNodesReady(c *client.Client) error { + nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) + Expect(err).NotTo(HaveOccurred()) + var notReady []api.Node + for _, node := range nodes.Items { + if isNodeReadySetAsExpected(&node, false) { + notReady = append(notReady, node) + } + } + if len(notReady) > 0 { + return fmt.Errorf("Not ready nodes: %v", notReady) + } + return nil +} + // Filters nodes in NodeList in place, removing nodes that do not // satisfy the given condition // TODO: consider merging with pkg/client/cache.NodeLister From 9854d4792331f43eec8e9e3deb7d247a2dad0bc5 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Jul 2015 13:54:22 +0200 Subject: [PATCH 2/2] Move Proxy logs out of flaky --- hack/jenkins/e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index d994e906065..3e79dcf572d 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -69,12 +69,12 @@ fi GCE_DEFAULT_SKIP_TEST_REGEX="Skipped|Density|Reboot|Restart|Example" # The following tests are known to be flaky, and are thus run only in their own # -flaky- build variants. -GCE_FLAKY_TEST_REGEX="Elasticsearch|Proxy.*logs|Shell.*services|MaxPods.*" +GCE_FLAKY_TEST_REGEX="Elasticsearch|Shell.*services|MaxPods.*" # Tests which are not able to be run in parallel. GCE_PARALLEL_SKIP_TEST_REGEX="${GCE_DEFAULT_SKIP_TEST_REGEX}|Etcd|NetworkingNew|Nodes\sNetwork|Nodes\sResize|MaxPods" # Tests which are known to be flaky when run in parallel. # TODO: figure out why GCE_FLAKY_TEST_REGEX is not a perfect subset of this list. -GCE_PARALLEL_FLAKY_TEST_REGEX="Addon|Elasticsearch|Hostdir.*MOD|Networking.*intra|PD|Proxy.*logs|ServiceAccounts|Service\sendpoints\slatency|Services.*change\sthe\stype|Services.*functioning\sexternal\sload\sbalancer|Services.*identically\snamed|Services.*release.*load\sbalancer|Shell|multiport\sendpoints" +GCE_PARALLEL_FLAKY_TEST_REGEX="Addon|Elasticsearch|Hostdir.*MOD|Networking.*intra|PD|ServiceAccounts|Service\sendpoints\slatency|Services.*change\sthe\stype|Services.*functioning\sexternal\sload\sbalancer|Services.*identically\snamed|Services.*release.*load\sbalancer|Shell|multiport\sendpoints" # Define environment variables based on the Jenkins project name. case ${JOB_NAME} in