Merge pull request #14646 from mesosphere/sttts-fix-kubeproxy-test

Fix e2e tests for providers without SSH to the nodes

cluster/mesos/docker/docker-compose.yml

@@ -47,7 +47,7 @@ mesosslave:
   - MESOS_PORT=5051
   - MESOS_LOG_DIR=/var/log/mesos
   - MESOS_LOGGING_LEVEL=INFO
-  - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099]
+  - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[8000-21099]
   - MESOS_SWITCH_USER=0
   - MESOS_CONTAINERIZERS=docker,mesos
   - DOCKER_DAEMON_ARGS

test/e2e/core.go

@@ -38,7 +38,7 @@ func CoreDump(dir string) {
 	provider := testContext.Provider
 
 	// requires ssh
-	if !providerIs("gce", "gke") {
+	if !providerIs(providersWithSSH...) {
 		fmt.Printf("Skipping SSH core dump, which is not implemented for %s", provider)
 		return
 	}

test/e2e/kubeproxy.go

@@ -68,7 +68,10 @@ var _ = Describe("KubeProxy", func() {
 	config := &KubeProxyTestConfig{
 		f: f,
 	}
+
 	It("should test kube-proxy", func() {
+		SkipUnlessProviderIs(providersWithSSH...)
+
 		By("cleaning up any pre-existing namespaces used by this test")
 		config.cleanup()
 
@@ -164,7 +167,7 @@ func (config *KubeProxyTestConfig) hitClusterIP(epCount int) {
 }
 
 func (config *KubeProxyTestConfig) hitNodePort(epCount int) {
-	node1_IP := strings.TrimSuffix(config.nodes[0], ":22")
+	node1_IP := config.nodes[0]
 	tries := epCount*epCount + 5 // + 10 if epCount == 0
 	By("dialing(udp) node1 --> node1:nodeUdpPort")
 	config.dialFromNode("udp", node1_IP, nodeUdpPort, tries, epCount)
@@ -188,7 +191,7 @@ func (config *KubeProxyTestConfig) hitNodePort(epCount int) {
 	By("Test disabled. dialing(http) node --> 127.0.0.1:nodeHttpPort")
 	//config.dialFromNode("http", "127.0.0.1", nodeHttpPort, tries, epCount)
 
-	node2_IP := strings.TrimSuffix(config.nodes[1], ":22")
+	node2_IP := config.nodes[1]
 	By("dialing(udp) node1 --> node2:nodeUdpPort")
 	config.dialFromNode("udp", node2_IP, nodeUdpPort, tries, epCount)
 	By("dialing(http) node1 --> node2:nodeHttpPort")
@@ -249,7 +252,7 @@ func (config *KubeProxyTestConfig) dialFromNode(protocol, targetIP string, targe
 }
 
 func (config *KubeProxyTestConfig) ssh(cmd string) string {
-	stdout, _, code, err := SSH(cmd, config.nodes[0], testContext.Provider)
+	stdout, _, code, err := SSH(cmd, config.nodes[0]+":22", testContext.Provider)
 	Expect(err).NotTo(HaveOccurred(), "error while SSH-ing to node: %v (code %v)", err, code)
 	Expect(code).Should(BeZero(), "command exited with non-zero code %v. cmd:%s", code, cmd)
 	return stdout
@@ -421,10 +424,10 @@ func (config *KubeProxyTestConfig) setup() {
 	By("Getting ssh-able hosts")
 	hosts, err := NodeSSHHosts(config.f.Client)
 	Expect(err).NotTo(HaveOccurred())
-	if len(hosts) == 0 {
+	config.nodes = make([]string, 0, len(hosts))
-		Failf("No ssh-able nodes")
+	for _, h := range hosts {
+		config.nodes = append(config.nodes, strings.TrimSuffix(h, ":22"))
 	}
-	config.nodes = hosts
 
 	if enableLoadBalancerTest {
 		By("Creating the LoadBalancer Service on top of the pods in kubernetes")

test/e2e/privileged.go

@@ -52,6 +52,7 @@ var _ = Describe("PrivilegedPod", func() {
 		f: f,
 	}
 	It("should test privileged pod", func() {
+		SkipUnlessProviderIs(providersWithSSH...)
 
 		By("Getting ssh-able hosts")
 		hosts, err := NodeSSHHosts(config.f.Client)

test/e2e/service.go

@@ -238,7 +238,7 @@ var _ = Describe("Services", func() {
 
 	It("should be able to up and down services", func() {
 		// this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP
-		SkipUnlessProviderIs("gce", "gke", "aws")
+		SkipUnlessProviderIs(providersWithSSH...)
 		ns := namespaces[0]
 		numPods, servicePort := 3, 80
 
@@ -532,14 +532,17 @@ var _ = Describe("Services", func() {
 		ip := pickNodeIP(c)
 		testReachable(ip, nodePort)
 
-		hosts, err := NodeSSHHosts(c)
+		// this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP
-		if err != nil {
+		if providerIs(providersWithSSH...) {
-			Expect(err).NotTo(HaveOccurred())
+			hosts, err := NodeSSHHosts(c)
-		}
+			if err != nil {
-		cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort)
+				Expect(err).NotTo(HaveOccurred())
-		_, _, code, err := SSH(cmd, hosts[0], testContext.Provider)
+			}
-		if code != 0 {
+			cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort)
-			Failf("expected node port (%d) to be in use", nodePort)
+			_, _, code, err := SSH(cmd, hosts[0], testContext.Provider)
+			if code != 0 {
+				Failf("expected node port (%d) to be in use", nodePort)
+			}
 		}
 	})
 
@@ -965,14 +968,17 @@ var _ = Describe("Services", func() {
 		err = t.DeleteService(serviceName)
 		Expect(err).NotTo(HaveOccurred())
 
-		hosts, err := NodeSSHHosts(c)
+		// this test uses NodeSSHHosts that does not work if a Node only reports LegacyHostIP
-		if err != nil {
+		if providerIs(providersWithSSH...) {
-			Expect(err).NotTo(HaveOccurred())
+			hosts, err := NodeSSHHosts(c)
-		}
+			if err != nil {
-		cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort)
+				Expect(err).NotTo(HaveOccurred())
-		_, _, code, err := SSH(cmd, hosts[0], testContext.Provider)
+			}
-		if code == 0 {
+			cmd := fmt.Sprintf(`test -n "$(ss -ant46 'sport = :%d' | tail -n +2 | grep LISTEN)"`, nodePort)
-			Failf("expected node port (%d) to not be in use", nodePort)
+			_, _, code, err := SSH(cmd, hosts[0], testContext.Provider)
+			if code == 0 {
+				Failf("expected node port (%d) to not be in use", nodePort)
+			}
 		}
 
 		By(fmt.Sprintf("creating service "+serviceName+" with same NodePort %d", nodePort))

test/e2e/ssh.go

@@ -35,7 +35,7 @@ var _ = Describe("SSH", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		// When adding more providers here, also implement their functionality in util.go's getSigner(...).
-		SkipUnlessProviderIs("gce", "gke")
+		SkipUnlessProviderIs(providersWithSSH...)
 	})
 
 	It("should SSH to all nodes and run commands", func() {

test/e2e/util.go

@@ -242,6 +242,9 @@ func providerIs(providers ...string) bool {
 	return false
 }
 
+// providersWithSSH are those providers where each node is accessible with SSH
+var providersWithSSH = []string{"gce", "gke", "aws"}
+
 type podCondition func(pod *api.Pod) (bool, error)
 
 // podReady returns whether pod has a condition of Ready with a status of true.