From a68e0848dc6927316e3e14a95031958d1ec35b94 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 29 Sep 2015 10:53:54 +0200 Subject: [PATCH 1/4] Factor out providers which support SSH in e2e tests --- test/e2e/core.go | 2 +- test/e2e/service.go | 2 +- test/e2e/ssh.go | 2 +- test/e2e/util.go | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e/core.go b/test/e2e/core.go index 2d1c42b754e..19f4c857386 100644 --- a/test/e2e/core.go +++ b/test/e2e/core.go @@ -38,7 +38,7 @@ func CoreDump(dir string) { provider := testContext.Provider // requires ssh - if !providerIs("gce", "gke") { + if !providerIs(providersWithSSH...) { fmt.Printf("Skipping SSH core dump, which is not implemented for %s", provider) return } diff --git a/test/e2e/service.go b/test/e2e/service.go index e6ae07a0fd2..e8d4fbcd6ae 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -238,7 +238,7 @@ var _ = Describe("Services", func() { It("should be able to up and down services", func() { // this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP - SkipUnlessProviderIs("gce", "gke", "aws") + SkipUnlessProviderIs(providersWithSSH...) ns := namespaces[0] numPods, servicePort := 3, 80 diff --git a/test/e2e/ssh.go b/test/e2e/ssh.go index a78101c6156..2efdf36af18 100644 --- a/test/e2e/ssh.go +++ b/test/e2e/ssh.go @@ -35,7 +35,7 @@ var _ = Describe("SSH", func() { Expect(err).NotTo(HaveOccurred()) // When adding more providers here, also implement their functionality in util.go's getSigner(...). - SkipUnlessProviderIs("gce", "gke") + SkipUnlessProviderIs(providersWithSSH...) }) It("should SSH to all nodes and run commands", func() { diff --git a/test/e2e/util.go b/test/e2e/util.go index 92f89b4ef74..b1b9a8f0187 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -241,6 +241,9 @@ func providerIs(providers ...string) bool { return false } +// providersWithSSH are those providers where each node is accessible with SSH +var providersWithSSH = []string{"gce", "gke", "aws"} + type podCondition func(pod *api.Pod) (bool, error) // podReady returns whether pod has a condition of Ready with a status of true. From 657db0a2b5cec39223152c5f1e1869ea10cfae76 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 28 Sep 2015 12:21:00 +0200 Subject: [PATCH 2/4] Add missing guards around SSH based e2e tests All other e2e tests which use SSH are skipped for providers other than gce, gke and aws. This patch does the same for - "should release NodePorts on delete" in service.go - "should test privileged pod" in privileged.go. --- test/e2e/privileged.go | 1 + test/e2e/service.go | 38 ++++++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/test/e2e/privileged.go b/test/e2e/privileged.go index e2cbf5f1ea4..b2cf98e08f1 100644 --- a/test/e2e/privileged.go +++ b/test/e2e/privileged.go @@ -52,6 +52,7 @@ var _ = Describe("PrivilegedPod", func() { f: f, } It("should test privileged pod", func() { + SkipUnlessProviderIs(providersWithSSH...) By("Getting ssh-able hosts") hosts, err := NodeSSHHosts(config.f.Client) diff --git a/test/e2e/service.go b/test/e2e/service.go index e8d4fbcd6ae..a31b38f8675 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -532,14 +532,17 @@ var _ = Describe("Services", func() { ip := pickNodeIP(c) testReachable(ip, nodePort) - hosts, err := NodeSSHHosts(c) - if err != nil { - Expect(err).NotTo(HaveOccurred()) - } - cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort) - _, _, code, err := SSH(cmd, hosts[0], testContext.Provider) - if code != 0 { - Failf("expected node port (%d) to be in use", nodePort) + // this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP + if providerIs(providersWithSSH...) { + hosts, err := NodeSSHHosts(c) + if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort) + _, _, code, err := SSH(cmd, hosts[0], testContext.Provider) + if code != 0 { + Failf("expected node port (%d) to be in use", nodePort) + } } }) @@ -965,14 +968,17 @@ var _ = Describe("Services", func() { err = t.DeleteService(serviceName) Expect(err).NotTo(HaveOccurred()) - hosts, err := NodeSSHHosts(c) - if err != nil { - Expect(err).NotTo(HaveOccurred()) - } - cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort) - _, _, code, err := SSH(cmd, hosts[0], testContext.Provider) - if code == 0 { - Failf("expected node port (%d) to not be in use", nodePort) + // this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP + if providerIs(providersWithSSH...) { + hosts, err := NodeSSHHosts(c) + if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort) + _, _, code, err := SSH(cmd, hosts[0], testContext.Provider) + if code == 0 { + Failf("expected node port (%d) to not be in use", nodePort) + } } By(fmt.Sprintf("creating service "+serviceName+" with same NodePort %d", nodePort)) From 5ecc2629b70e758953eca6cc12dc2a43896e8b43 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 28 Sep 2015 12:21:00 +0200 Subject: [PATCH 3/4] Refactor kubeproxy tests to run with and without provider SSH support All other e2e tests which use SSH are skipped for providers other than gce, gke and aws. This patch does the same for the kube-proxy test. If not SSH support is available for the provider, the test will still run, but use less probes. --- test/e2e/kubeproxy.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/e2e/kubeproxy.go b/test/e2e/kubeproxy.go index 2949d009a22..3f2d29032e4 100644 --- a/test/e2e/kubeproxy.go +++ b/test/e2e/kubeproxy.go @@ -68,7 +68,10 @@ var _ = Describe("KubeProxy", func() { config := &KubeProxyTestConfig{ f: f, } + It("should test kube-proxy", func() { + SkipUnlessProviderIs(providersWithSSH...) + By("cleaning up any pre-existing namespaces used by this test") config.cleanup() @@ -164,7 +167,7 @@ func (config *KubeProxyTestConfig) hitClusterIP(epCount int) { } func (config *KubeProxyTestConfig) hitNodePort(epCount int) { - node1_IP := strings.TrimSuffix(config.nodes[0], ":22") + node1_IP := config.nodes[0] tries := epCount*epCount + 5 // + 10 if epCount == 0 By("dialing(udp) node1 --> node1:nodeUdpPort") config.dialFromNode("udp", node1_IP, nodeUdpPort, tries, epCount) @@ -188,7 +191,7 @@ func (config *KubeProxyTestConfig) hitNodePort(epCount int) { By("Test disabled. dialing(http) node --> 127.0.0.1:nodeHttpPort") //config.dialFromNode("http", "127.0.0.1", nodeHttpPort, tries, epCount) - node2_IP := strings.TrimSuffix(config.nodes[1], ":22") + node2_IP := config.nodes[1] By("dialing(udp) node1 --> node2:nodeUdpPort") config.dialFromNode("udp", node2_IP, nodeUdpPort, tries, epCount) By("dialing(http) node1 --> node2:nodeHttpPort") @@ -249,7 +252,7 @@ func (config *KubeProxyTestConfig) dialFromNode(protocol, targetIP string, targe } func (config *KubeProxyTestConfig) ssh(cmd string) string { - stdout, _, code, err := SSH(cmd, config.nodes[0], testContext.Provider) + stdout, _, code, err := SSH(cmd, config.nodes[0]+":22", testContext.Provider) Expect(err).NotTo(HaveOccurred(), "error while SSH-ing to node: %v (code %v)", err, code) Expect(code).Should(BeZero(), "command exited with non-zero code %v. cmd:%s", code, cmd) return stdout @@ -421,10 +424,10 @@ func (config *KubeProxyTestConfig) setup() { By("Getting ssh-able hosts") hosts, err := NodeSSHHosts(config.f.Client) Expect(err).NotTo(HaveOccurred()) - if len(hosts) == 0 { - Failf("No ssh-able nodes") + config.nodes = make([]string, 0, len(hosts)) + for _, h := range hosts { + config.nodes = append(config.nodes, strings.TrimSuffix(h, ":22")) } - config.nodes = hosts if enableLoadBalancerTest { By("Creating the LoadBalancer Service on top of the pods in kubernetes") From b041d4ec1163c511c05e27a1c50c56cfa439c6e9 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 2 Oct 2015 16:37:02 +0200 Subject: [PATCH 4/4] Enlarge port range for mesos/docker e2e tests --- cluster/mesos/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml index 32ca8259cc8..08c6ef43844 100644 --- a/cluster/mesos/docker/docker-compose.yml +++ b/cluster/mesos/docker/docker-compose.yml @@ -47,7 +47,7 @@ mesosslave: - MESOS_PORT=5051 - MESOS_LOG_DIR=/var/log/mesos - MESOS_LOGGING_LEVEL=INFO - - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099] + - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[8000-21099] - MESOS_SWITCH_USER=0 - MESOS_CONTAINERIZERS=docker,mesos - DOCKER_DAEMON_ARGS