From 5b8e16d2558c2aed703f6af306d102bea9b0278e Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Thu, 29 Sep 2016 15:26:04 -0700 Subject: [PATCH] move pod networking tests common --- test/e2e/common/networking.go | 62 ++++++ test/e2e/common/util.go | 1 + test/e2e/networking.go | 152 +++++-------- test/e2e_node/remote/remote.go | 25 ++- test/{e2e => utils}/networking_utils.go | 272 ++++++++++++------------ 5 files changed, 285 insertions(+), 227 deletions(-) create mode 100644 test/e2e/common/networking.go rename test/{e2e => utils}/networking_utils.go (68%) diff --git a/test/e2e/common/networking.go b/test/e2e/common/networking.go new file mode 100644 index 00000000000..d861816abdb --- /dev/null +++ b/test/e2e/common/networking.go @@ -0,0 +1,62 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + . "github.com/onsi/ginkgo" + "k8s.io/kubernetes/pkg/util/sets" + "k8s.io/kubernetes/test/e2e/framework" + networking_util "k8s.io/kubernetes/test/utils" +) + +var _ = framework.KubeDescribe("Networking", func() { + f := framework.NewDefaultFramework("pod-network-test") + + framework.KubeDescribe("Granular Checks: Pods", func() { + + // Try to hit all endpoints through a test container, retry 5 times, + // expect exactly one unique hostname. Each of these endpoints reports + // its own hostname. + It("should function for intra-pod communication: http [Conformance]", func() { + config := networking_util.NewCoreNetworkingTestConfig(f) + for _, endpointPod := range config.EndpointPods { + config.DialFromTestContainer("http", endpointPod.Status.PodIP, networking_util.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + } + }) + + It("should function for intra-pod communication: udp [Conformance]", func() { + config := networking_util.NewCoreNetworkingTestConfig(f) + for _, endpointPod := range config.EndpointPods { + config.DialFromTestContainer("udp", endpointPod.Status.PodIP, networking_util.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + } + }) + + It("should function for node-pod communication: http [Conformance]", func() { + config := networking_util.NewCoreNetworkingTestConfig(f) + for _, endpointPod := range config.EndpointPods { + config.DialFromNode("http", endpointPod.Status.PodIP, networking_util.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + } + }) + + It("should function for node-pod communication: udp [Conformance]", func() { + config := networking_util.NewCoreNetworkingTestConfig(f) + for _, endpointPod := range config.EndpointPods { + config.DialFromNode("udp", endpointPod.Status.PodIP, networking_util.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) + } + }) + }) +}) diff --git a/test/e2e/common/util.go b/test/e2e/common/util.go index cc4f64efb04..fdde2d28ff5 100644 --- a/test/e2e/common/util.go +++ b/test/e2e/common/util.go @@ -40,6 +40,7 @@ var CommonImageWhiteList = sets.NewString( "gcr.io/google_containers/mounttest:0.7", "gcr.io/google_containers/mounttest-user:0.3", "gcr.io/google_containers/netexec:1.4", + "gcr.io/google_containers/netexec:1.5", "gcr.io/google_containers/nginx-slim:0.7", "gcr.io/google_containers/serve_hostname:v1.4", "gcr.io/google_containers/test-webserver:e2e", diff --git a/test/e2e/networking.go b/test/e2e/networking.go index cb74b26cb84..46e7cad5563 100644 --- a/test/e2e/networking.go +++ b/test/e2e/networking.go @@ -20,10 +20,10 @@ import ( "fmt" "net/http" + . "github.com/onsi/ginkgo" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/ginkgo" + networking_util "k8s.io/kubernetes/test/utils" ) var _ = framework.KubeDescribe("Networking", func() { @@ -77,148 +77,114 @@ var _ = framework.KubeDescribe("Networking", func() { It("should check kube-proxy urls", func() { // TODO: this is overkill we just need the host networking pod // to hit kube-proxy urls. - config := NewNetworkingTestConfig(f) + config := networking_util.NewNetworkingTestConfig(f) By("checking kube-proxy URLs") - config.getSelfURL("/healthz", "ok") - config.getSelfURL("/proxyMode", "iptables") // the default - }) - - framework.KubeDescribe("Granular Checks: Pods", func() { - - // Try to hit all endpoints through a test container, retry 5 times, - // expect exactly one unique hostname. Each of these endpoints reports - // its own hostname. - It("should function for intra-pod communication: http [Conformance]", func() { - config := NewNetworkingTestConfig(f) - for _, endpointPod := range config.endpointPods { - config.dialFromTestContainer("http", endpointPod.Status.PodIP, endpointHttpPort, config.maxTries, 0, sets.NewString(endpointPod.Name)) - } - }) - - It("should function for intra-pod communication: udp [Conformance]", func() { - config := NewNetworkingTestConfig(f) - for _, endpointPod := range config.endpointPods { - config.dialFromTestContainer("udp", endpointPod.Status.PodIP, endpointUdpPort, config.maxTries, 0, sets.NewString(endpointPod.Name)) - } - }) - - It("should function for node-pod communication: http [Conformance]", func() { - config := NewNetworkingTestConfig(f) - for _, endpointPod := range config.endpointPods { - config.dialFromNode("http", endpointPod.Status.PodIP, endpointHttpPort, config.maxTries, 0, sets.NewString(endpointPod.Name)) - } - }) - - It("should function for node-pod communication: udp [Conformance]", func() { - config := NewNetworkingTestConfig(f) - for _, endpointPod := range config.endpointPods { - config.dialFromNode("udp", endpointPod.Status.PodIP, endpointUdpPort, config.maxTries, 0, sets.NewString(endpointPod.Name)) - } - }) + config.GetSelfURL("/healthz", "ok") + config.GetSelfURL("/proxyMode", "iptables") // the default }) // TODO: Remove [Slow] when this has had enough bake time to prove presubmit worthiness. framework.KubeDescribe("Granular Checks: Services [Slow]", func() { It("should function for pod-Service: http", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterHttpPort)) - config.dialFromTestContainer("http", config.clusterIP, clusterHttpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterHttpPort)) + config.DialFromTestContainer("http", config.ClusterIP, networking_util.ClusterHttpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.testContainerPod.Name, config.externalAddrs[0], config.nodeHttpPort)) - config.dialFromTestContainer("http", config.nodeIP, config.nodeHttpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.ExternalAddrs[0], config.NodeHttpPort)) + config.DialFromTestContainer("http", config.NodeIP, config.NodeHttpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should function for pod-Service: udp", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterUdpPort)) - config.dialFromTestContainer("udp", config.clusterIP, clusterUdpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterUdpPort)) + config.DialFromTestContainer("udp", config.ClusterIP, networking_util.ClusterUdpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.testContainerPod.Name, config.externalAddrs[0], config.nodeUdpPort)) - config.dialFromTestContainer("udp", config.nodeIP, config.nodeUdpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.ExternalAddrs[0], config.NodeUdpPort)) + config.DialFromTestContainer("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should function for node-Service: http", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.nodeIP, config.clusterIP, clusterHttpPort)) - config.dialFromNode("http", config.clusterIP, clusterHttpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.NodeIP, config.ClusterIP, networking_util.ClusterHttpPort)) + config.DialFromNode("http", config.ClusterIP, networking_util.ClusterHttpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeHttpPort)) - config.dialFromNode("http", config.nodeIP, config.nodeHttpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHttpPort)) + config.DialFromNode("http", config.NodeIP, config.NodeHttpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should function for node-Service: udp", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.nodeIP, config.clusterIP, clusterUdpPort)) - config.dialFromNode("udp", config.clusterIP, clusterUdpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.NodeIP, config.ClusterIP, networking_util.ClusterUdpPort)) + config.DialFromNode("udp", config.ClusterIP, networking_util.ClusterUdpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeUdpPort)) - config.dialFromNode("udp", config.nodeIP, config.nodeUdpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUdpPort)) + config.DialFromNode("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should function for endpoint-Service: http", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.endpointPods[0].Name, config.clusterIP, clusterHttpPort)) - config.dialFromEndpointContainer("http", config.clusterIP, clusterHttpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.ClusterIP, networking_util.ClusterHttpPort)) + config.DialFromEndpointContainer("http", config.ClusterIP, networking_util.ClusterHttpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.endpointPods[0].Name, config.nodeIP, config.nodeHttpPort)) - config.dialFromEndpointContainer("http", config.nodeIP, config.nodeHttpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.NodeIP, config.NodeHttpPort)) + config.DialFromEndpointContainer("http", config.NodeIP, config.NodeHttpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should function for endpoint-Service: udp", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.endpointPods[0].Name, config.clusterIP, clusterUdpPort)) - config.dialFromEndpointContainer("udp", config.clusterIP, clusterUdpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.ClusterIP, networking_util.ClusterUdpPort)) + config.DialFromEndpointContainer("udp", config.ClusterIP, networking_util.ClusterUdpPort, config.MaxTries, 0, config.EndpointHostnames()) - By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.endpointPods[0].Name, config.nodeIP, config.nodeUdpPort)) - config.dialFromEndpointContainer("udp", config.nodeIP, config.nodeUdpPort, config.maxTries, 0, config.endpointHostnames()) + By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.NodeIP, config.NodeUdpPort)) + config.DialFromEndpointContainer("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, 0, config.EndpointHostnames()) }) It("should update endpoints: http", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterHttpPort)) - config.dialFromTestContainer("http", config.clusterIP, clusterHttpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterHttpPort)) + config.DialFromTestContainer("http", config.ClusterIP, networking_util.ClusterHttpPort, config.MaxTries, 0, config.EndpointHostnames()) - config.deleteNetProxyPod() + config.DeleteNetProxyPod() - By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterHttpPort)) - config.dialFromTestContainer("http", config.clusterIP, clusterHttpPort, config.maxTries, config.maxTries, config.endpointHostnames()) + By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterHttpPort)) + config.DialFromTestContainer("http", config.ClusterIP, networking_util.ClusterHttpPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) }) It("should update endpoints: udp", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterUdpPort)) - config.dialFromTestContainer("udp", config.clusterIP, clusterUdpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterUdpPort)) + config.DialFromTestContainer("udp", config.ClusterIP, networking_util.ClusterUdpPort, config.MaxTries, 0, config.EndpointHostnames()) - config.deleteNetProxyPod() + config.DeleteNetProxyPod() - By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.testContainerPod.Name, config.clusterIP, clusterUdpPort)) - config.dialFromTestContainer("udp", config.clusterIP, clusterUdpPort, config.maxTries, config.maxTries, config.endpointHostnames()) + By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, networking_util.ClusterUdpPort)) + config.DialFromTestContainer("udp", config.ClusterIP, networking_util.ClusterUdpPort, config.MaxTries, config.MaxTries, config.EndpointHostnames()) }) // Slow because we confirm that the nodePort doesn't serve traffic, which requires a period of polling. It("should update nodePort: http [Slow]", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeHttpPort)) - config.dialFromNode("http", config.nodeIP, config.nodeHttpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHttpPort)) + config.DialFromNode("http", config.NodeIP, config.NodeHttpPort, config.MaxTries, 0, config.EndpointHostnames()) - config.deleteNodePortService() + config.DeleteNodePortService() - By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeHttpPort)) - config.dialFromNode("http", config.nodeIP, config.nodeHttpPort, config.maxTries, config.maxTries, sets.NewString()) + By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeHttpPort)) + config.DialFromNode("http", config.NodeIP, config.NodeHttpPort, config.MaxTries, config.MaxTries, sets.NewString()) }) // Slow because we confirm that the nodePort doesn't serve traffic, which requires a period of polling. It("should update nodePort: udp [Slow]", func() { - config := NewNetworkingTestConfig(f) - By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeUdpPort)) - config.dialFromNode("udp", config.nodeIP, config.nodeUdpPort, config.maxTries, 0, config.endpointHostnames()) + config := networking_util.NewNetworkingTestConfig(f) + By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUdpPort)) + config.DialFromNode("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, 0, config.EndpointHostnames()) - config.deleteNodePortService() + config.DeleteNodePortService() - By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.nodeIP, config.nodeIP, config.nodeUdpPort)) - config.dialFromNode("udp", config.nodeIP, config.nodeUdpPort, config.maxTries, config.maxTries, sets.NewString()) + By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUdpPort)) + config.DialFromNode("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, config.MaxTries, sets.NewString()) }) // TODO: Test sessionAffinity #31712 }) diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index 2c77b9ecbf1..d056752ad32 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -165,6 +165,27 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string return "", false, err } + // Configure iptables firewall rules + // TODO: consider calling bootstrap script to configure host based on OS + cmd := getSshCommand("&&", + `iptables -L INPUT | grep "Chain INPUT (policy DROP)"`, + "(iptables -C INPUT -w -p TCP -j ACCEPT || iptables -A INPUT -w -p TCP -j ACCEPT)", + "(iptables -C INPUT -w -p UDP -j ACCEPT || iptables -A INPUT -w -p UDP -j ACCEPT)", + "(iptables -C INPUT -w -p ICMP -j ACCEPT || iptables -A INPUT -w -p ICMP -j ACCEPT)") + output, err := RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sudo", "sh", "-c", cmd) + if err != nil { + glog.Errorf("Failed to configured firewall: %v output: %v", err, output) + } + cmd = getSshCommand("&&", + `iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null`, + "(iptables -C FORWARD -w -p TCP -j ACCEPT || iptables -A FORWARD -w -p TCP -j ACCEPT)", + "(iptables -C FORWARD -w -p UDP -j ACCEPT || iptables -A FORWARD -w -p UDP -j ACCEPT)", + "(iptables -C FORWARD -w -p ICMP -j ACCEPT || iptables -A FORWARD -w -p ICMP -j ACCEPT)") + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sudo", "sh", "-c", cmd) + if err != nil { + glog.Errorf("Failed to configured firewall: %v output: %v", err, output) + } + // Copy the archive to the staging directory _, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), tmp)) if err != nil { @@ -173,7 +194,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string } // Kill any running node processes - cmd := getSshCommand(" ; ", + cmd = getSshCommand(" ; ", "sudo pkill kubelet", "sudo pkill kube-apiserver", "sudo pkill etcd", @@ -187,7 +208,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Extract the archive cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), fmt.Sprintf("tar -xzvf ./%s", archiveName)) glog.Infof("Extracting tar on %s", host) - output, err := RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", cmd) + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", cmd) if err != nil { // Exit failure with the error return "", false, err diff --git a/test/e2e/networking_utils.go b/test/utils/networking_utils.go similarity index 68% rename from test/e2e/networking_utils.go rename to test/utils/networking_utils.go index cf2b56e3de3..c2caa5b44c3 100644 --- a/test/e2e/networking_utils.go +++ b/test/utils/networking_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package utils import ( "encoding/json" @@ -38,13 +38,13 @@ import ( ) const ( - endpointHttpPort = 8080 - endpointUdpPort = 8081 - testContainerHttpPort = 8080 - clusterHttpPort = 80 - clusterUdpPort = 90 - netexecImageName = "gcr.io/google_containers/netexec:1.5" - hostexecImageName = "gcr.io/google_containers/hostexec:1.2" + EndpointHttpPort = 8080 + EndpointUdpPort = 8081 + TestContainerHttpPort = 8080 + ClusterHttpPort = 80 + ClusterUdpPort = 90 + NetexecImageName = "gcr.io/google_containers/netexec:1.5" + HostexecImageName = "gcr.io/google_containers/hostexec:1.2" testPodName = "test-container-pod" hostTestPodName = "host-test-container-pod" nodePortServiceName = "node-port-service" @@ -58,61 +58,79 @@ const ( // NewNetworkingTestConfig creates and sets up a new test config helper. func NewNetworkingTestConfig(f *framework.Framework) *NetworkingTestConfig { - config := &NetworkingTestConfig{f: f, ns: f.Namespace.Name} - By(fmt.Sprintf("Performing setup for networking test in namespace %v", config.ns)) - config.setup() + config := &NetworkingTestConfig{f: f, Namespace: f.Namespace.Name} + By(fmt.Sprintf("Performing setup for networking test in namespace %v", config.Namespace)) + config.setup(getServiceSelector()) return config } +// NewNetworkingTestNodeE2EConfig creates and sets up a new test config helper for Node E2E. +func NewCoreNetworkingTestConfig(f *framework.Framework) *NetworkingTestConfig { + config := &NetworkingTestConfig{f: f, Namespace: f.Namespace.Name} + By(fmt.Sprintf("Performing setup for networking test in namespace %v", config.Namespace)) + config.setupCore(getServiceSelector()) + return config +} + +func getServiceSelector() map[string]string { + By("creating a selector") + selectorName := "selector-" + string(uuid.NewUUID()) + serviceSelector := map[string]string{ + selectorName: "true", + } + return serviceSelector +} + // NetworkingTestConfig is a convenience class around some utility methods // for testing kubeproxy/networking/services/endpoints. type NetworkingTestConfig struct { - // testContaienrPod is a test pod running the netexec image. It is capable + // TestContaienrPod is a test pod running the netexec image. It is capable // of executing tcp/udp requests against ip:port. - testContainerPod *api.Pod - // hostTestContainerPod is a pod running with hostNetworking=true, and the + TestContainerPod *api.Pod + // HostTestContainerPod is a pod running with hostNetworking=true, and the // hostexec image. - hostTestContainerPod *api.Pod - // endpointPods are the pods belonging to the Service created by this + HostTestContainerPod *api.Pod + // EndpointPods are the pods belonging to the Service created by this // test config. Each invocation of `setup` creates a service with // 1 pod per node running the netexecImage. - endpointPods []*api.Pod + EndpointPods []*api.Pod f *framework.Framework - // nodePortService is a Service with Type=NodePort spanning over all + podClient *framework.PodClient + // NodePortService is a Service with Type=NodePort spanning over all // endpointPods. - nodePortService *api.Service - // externalAddrs is a list of external IPs of nodes in the cluster. - externalAddrs []string - // nodes is a list of nodes in the cluster. - nodes []api.Node - // maxTries is the number of retries tolerated for tests run against + NodePortService *api.Service + // ExternalAddrs is a list of external IPs of nodes in the cluster. + ExternalAddrs []string + // Nodes is a list of nodes in the cluster. + Nodes []api.Node + // MaxTries is the number of retries tolerated for tests run against // endpoints and services created by this config. - maxTries int - // The clusterIP of the Service reated by this test config. - clusterIP string + MaxTries int + // The ClusterIP of the Service reated by this test config. + ClusterIP string // External ip of first node for use in nodePort testing. - nodeIP string + NodeIP string // The http/udp nodePorts of the Service. - nodeHttpPort int - nodeUdpPort int + NodeHttpPort int + NodeUdpPort int // The kubernetes namespace within which all resources for this // config are created - ns string + Namespace string } -func (config *NetworkingTestConfig) dialFromEndpointContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { - config.dialFromContainer(protocol, config.endpointPods[0].Status.PodIP, targetIP, endpointHttpPort, targetPort, maxTries, minTries, expectedEps) +func (config *NetworkingTestConfig) DialFromEndpointContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { + config.DialFromContainer(protocol, config.EndpointPods[0].Status.PodIP, targetIP, EndpointHttpPort, targetPort, maxTries, minTries, expectedEps) } -func (config *NetworkingTestConfig) dialFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { - config.dialFromContainer(protocol, config.testContainerPod.Status.PodIP, targetIP, testContainerHttpPort, targetPort, maxTries, minTries, expectedEps) +func (config *NetworkingTestConfig) DialFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { + config.DialFromContainer(protocol, config.TestContainerPod.Status.PodIP, targetIP, TestContainerHttpPort, targetPort, maxTries, minTries, expectedEps) } // diagnoseMissingEndpoints prints debug information about the endpoints that // are NOT in the given list of foundEndpoints. These are the endpoints we // expected a response from. func (config *NetworkingTestConfig) diagnoseMissingEndpoints(foundEndpoints sets.String) { - for _, e := range config.endpointPods { + for _, e := range config.EndpointPods { if foundEndpoints.Has(e.Name) { continue } @@ -123,16 +141,16 @@ func (config *NetworkingTestConfig) diagnoseMissingEndpoints(foundEndpoints sets } } -// endpointHostnames returns a set of hostnames for existing endpoints. -func (config *NetworkingTestConfig) endpointHostnames() sets.String { +// EndpointHostnames returns a set of hostnames for existing endpoints. +func (config *NetworkingTestConfig) EndpointHostnames() sets.String { expectedEps := sets.NewString() - for _, p := range config.endpointPods { + for _, p := range config.EndpointPods { expectedEps.Insert(p.Name) } return expectedEps } -// dialFromContainers executes a curl via kubectl exec in a test container, +// DialFromContainers executes a curl via kubectl exec in a test container, // which might then translate to a tcp or udp request based on the protocol // argument in the url. // - minTries is the minimum number of curl attempts required before declaring @@ -145,7 +163,7 @@ func (config *NetworkingTestConfig) endpointHostnames() sets.String { // maxTries == minTries will confirm that we see the expected endpoints and no // more for maxTries. Use this if you want to eg: fail a readiness check on a // pod and confirm it doesn't show up as an endpoint. -func (config *NetworkingTestConfig) dialFromContainer(protocol, containerIP, targetIP string, containerHttpPort, targetPort, maxTries, minTries int, expectedEps sets.String) { +func (config *NetworkingTestConfig) DialFromContainer(protocol, containerIP, targetIP string, containerHttpPort, targetPort, maxTries, minTries int, expectedEps sets.String) { cmd := fmt.Sprintf("curl -q -s 'http://%s:%d/dial?request=hostName&protocol=%s&host=%s&port=%d&tries=1'", containerIP, containerHttpPort, @@ -156,23 +174,19 @@ func (config *NetworkingTestConfig) dialFromContainer(protocol, containerIP, tar eps := sets.NewString() for i := 0; i < maxTries; i++ { - stdout, err := framework.RunHostCmd(config.ns, config.hostTestContainerPod.Name, cmd) - if err != nil { - // A failure to kubectl exec counts as a try, not a hard fail. - // Also note that we will keep failing for maxTries in tests where - // we confirm unreachability. - framework.Logf("Failed to execute %v: %v", cmd, err) - } else { - var output map[string][]string - if err := json.Unmarshal([]byte(stdout), &output); err != nil { - framework.Logf("WARNING: Failed to unmarshal curl response. Cmd %v run in %v, output: %s, err: %v", - cmd, config.hostTestContainerPod.Name, stdout, err) - continue - } - for _, hostName := range output["responses"] { - eps.Insert(hostName) - } + stdout := config.f.ExecShellInPod(config.HostTestContainerPod.Name, cmd) + + var output map[string][]string + if err := json.Unmarshal([]byte(stdout), &output); err != nil { + framework.Logf("WARNING: Failed to unmarshal curl response. Cmd %v run in %v, output: %s, err: %v", + cmd, config.HostTestContainerPod.Name, stdout, err) + continue } + + for _, hostName := range output["responses"] { + eps.Insert(hostName) + } + framework.Logf("Waiting for endpoints: %v", expectedEps.Difference(eps)) // Check against i+1 so we exit if minTries == maxTries. @@ -185,7 +199,7 @@ func (config *NetworkingTestConfig) dialFromContainer(protocol, containerIP, tar framework.Failf("Failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", minTries, cmd, eps, expectedEps) } -// dialFromNode executes a tcp or udp request based on protocol via kubectl exec +// DialFromNode executes a tcp or udp request based on protocol via kubectl exec // in a test container running with host networking. // - minTries is the minimum number of curl attempts required before declaring // success. Set to 0 if you'd like to return as soon as all endpoints respond @@ -195,7 +209,7 @@ func (config *NetworkingTestConfig) dialFromContainer(protocol, containerIP, tar // maxTries == minTries will confirm that we see the expected endpoints and no // more for maxTries. Use this if you want to eg: fail a readiness check on a // pod and confirm it doesn't show up as an endpoint. -func (config *NetworkingTestConfig) dialFromNode(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { +func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { var cmd string if protocol == "udp" { cmd = fmt.Sprintf("echo 'hostName' | timeout -t 3 nc -w 1 -u %s %d", targetIP, targetPort) @@ -210,15 +224,8 @@ func (config *NetworkingTestConfig) dialFromNode(protocol, targetIP string, targ filterCmd := fmt.Sprintf("%s | grep -v '^\\s*$'", cmd) for i := 0; i < maxTries; i++ { - stdout, err := framework.RunHostCmd(config.ns, config.hostTestContainerPod.Name, filterCmd) - if err != nil { - // A failure to kubectl exec counts as a try, not a hard fail. - // Also note that we will keep failing for maxTries in tests where - // we confirm unreachability. - framework.Logf("Failed to execute %v: %v", filterCmd, err) - } else { - eps.Insert(strings.TrimSpace(stdout)) - } + stdout := config.f.ExecShellInPod(config.HostTestContainerPod.Name, filterCmd) + eps.Insert(strings.TrimSpace(stdout)) framework.Logf("Waiting for %+v endpoints, got endpoints %+v", expectedEps.Difference(eps), eps) // Check against i+1 so we exit if minTries == maxTries. @@ -231,13 +238,13 @@ func (config *NetworkingTestConfig) dialFromNode(protocol, targetIP string, targ framework.Failf("Failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", minTries, cmd, eps, expectedEps) } -// getSelfURL executes a curl against the given path via kubectl exec into a +// GetSelfURL executes a curl against the given path via kubectl exec into a // test container running with host networking, and fails if the output // doesn't match the expected string. -func (config *NetworkingTestConfig) getSelfURL(path string, expected string) { +func (config *NetworkingTestConfig) GetSelfURL(path string, expected string) { cmd := fmt.Sprintf("curl -q -s --connect-timeout 1 http://localhost:10249%s", path) By(fmt.Sprintf("Getting kube-proxy self URL %s", path)) - stdout := framework.RunHostCmdOrDie(config.ns, config.hostTestContainerPod.Name, cmd) + stdout := framework.RunHostCmdOrDie(config.Namespace, config.HostTestContainerPod.Name, cmd) Expect(strings.Contains(stdout, expected)).To(BeTrue()) } @@ -251,7 +258,7 @@ func (config *NetworkingTestConfig) createNetShellPodSpec(podName string, node s Handler: api.Handler{ HTTPGet: &api.HTTPGetAction{ Path: "/healthz", - Port: intstr.IntOrString{IntVal: endpointHttpPort}, + Port: intstr.IntOrString{IntVal: EndpointHttpPort}, }, }, } @@ -262,27 +269,27 @@ func (config *NetworkingTestConfig) createNetShellPodSpec(podName string, node s }, ObjectMeta: api.ObjectMeta{ Name: podName, - Namespace: config.ns, + Namespace: config.Namespace, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "webserver", - Image: netexecImageName, + Image: NetexecImageName, ImagePullPolicy: api.PullIfNotPresent, Command: []string{ "/netexec", - fmt.Sprintf("--http-port=%d", endpointHttpPort), - fmt.Sprintf("--udp-port=%d", endpointUdpPort), + fmt.Sprintf("--http-port=%d", EndpointHttpPort), + fmt.Sprintf("--udp-port=%d", EndpointUdpPort), }, Ports: []api.ContainerPort{ { Name: "http", - ContainerPort: endpointHttpPort, + ContainerPort: EndpointHttpPort, }, { Name: "udp", - ContainerPort: endpointUdpPort, + ContainerPort: EndpointUdpPort, Protocol: api.ProtocolUDP, }, }, @@ -304,23 +311,23 @@ func (config *NetworkingTestConfig) createTestPodSpec() *api.Pod { }, ObjectMeta: api.ObjectMeta{ Name: testPodName, - Namespace: config.ns, + Namespace: config.Namespace, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "webserver", - Image: netexecImageName, + Image: NetexecImageName, ImagePullPolicy: api.PullIfNotPresent, Command: []string{ "/netexec", - fmt.Sprintf("--http-port=%d", endpointHttpPort), - fmt.Sprintf("--udp-port=%d", endpointUdpPort), + fmt.Sprintf("--http-port=%d", EndpointHttpPort), + fmt.Sprintf("--udp-port=%d", EndpointUdpPort), }, Ports: []api.ContainerPort{ { Name: "http", - ContainerPort: testContainerHttpPort, + ContainerPort: TestContainerHttpPort, }, }, }, @@ -338,24 +345,24 @@ func (config *NetworkingTestConfig) createNodePortService(selector map[string]st Spec: api.ServiceSpec{ Type: api.ServiceTypeNodePort, Ports: []api.ServicePort{ - {Port: clusterHttpPort, Name: "http", Protocol: api.ProtocolTCP, TargetPort: intstr.FromInt(endpointHttpPort)}, - {Port: clusterUdpPort, Name: "udp", Protocol: api.ProtocolUDP, TargetPort: intstr.FromInt(endpointUdpPort)}, + {Port: ClusterHttpPort, Name: "http", Protocol: api.ProtocolTCP, TargetPort: intstr.FromInt(EndpointHttpPort)}, + {Port: ClusterUdpPort, Name: "udp", Protocol: api.ProtocolUDP, TargetPort: intstr.FromInt(EndpointUdpPort)}, }, Selector: selector, }, } - config.nodePortService = config.createService(serviceSpec) + config.NodePortService = config.createService(serviceSpec) } -func (config *NetworkingTestConfig) deleteNodePortService() { - err := config.getServiceClient().Delete(config.nodePortService.Name) +func (config *NetworkingTestConfig) DeleteNodePortService() { + err := config.getServiceClient().Delete(config.NodePortService.Name) Expect(err).NotTo(HaveOccurred(), "error while deleting NodePortService. err:%v)", err) time.Sleep(15 * time.Second) // wait for kube-proxy to catch up with the service being deleted. } func (config *NetworkingTestConfig) createTestPods() { testContainerPod := config.createTestPodSpec() - hostTestContainerPod := framework.NewHostExecPodSpec(config.ns, hostTestPodName) + hostTestContainerPod := framework.NewHostExecPodSpec(config.Namespace, hostTestPodName) config.createPod(testContainerPod) config.createPod(hostTestContainerPod) @@ -364,12 +371,12 @@ func (config *NetworkingTestConfig) createTestPods() { framework.ExpectNoError(config.f.WaitForPodRunning(hostTestContainerPod.Name)) var err error - config.testContainerPod, err = config.getPodClient().Get(testContainerPod.Name) + config.TestContainerPod, err = config.getPodClient().Get(testContainerPod.Name) if err != nil { framework.Failf("Failed to retrieve %s pod: %v", testContainerPod.Name, err) } - config.hostTestContainerPod, err = config.getPodClient().Get(hostTestContainerPod.Name) + config.HostTestContainerPod, err = config.getPodClient().Get(hostTestContainerPod.Name) if err != nil { framework.Failf("Failed to retrieve %s pod: %v", hostTestContainerPod.Name, err) } @@ -379,7 +386,7 @@ func (config *NetworkingTestConfig) createService(serviceSpec *api.Service) *api _, err := config.getServiceClient().Create(serviceSpec) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create %s service: %v", serviceSpec.Name, err)) - err = framework.WaitForService(config.f.Client, config.ns, serviceSpec.Name, true, 5*time.Second, 45*time.Second) + err = framework.WaitForService(config.f.Client, config.Namespace, serviceSpec.Name, true, 5*time.Second, 45*time.Second) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error while waiting for service:%s err: %v", serviceSpec.Name, err)) createdService, err := config.getServiceClient().Get(serviceSpec.Name) @@ -388,48 +395,50 @@ func (config *NetworkingTestConfig) createService(serviceSpec *api.Service) *api return createdService } -func (config *NetworkingTestConfig) setup() { - By("creating a selector") - selectorName := "selector-" + string(uuid.NewUUID()) - serviceSelector := map[string]string{ - selectorName: "true", - } +// setupCore sets up the pods and core test config +// mainly for simplified node e2e setup +func (config *NetworkingTestConfig) setupCore(selector map[string]string) { + By("Creating the service pods in kubernetes") + podName := "netserver" + config.EndpointPods = config.createNetProxyPods(podName, selector) + + By("Creating test pods") + config.createTestPods() + + epCount := len(config.EndpointPods) + config.MaxTries = epCount*epCount + testTries +} + +// setup includes setupCore and also sets up services +func (config *NetworkingTestConfig) setup(selector map[string]string) { + config.setupCore(selector) By("Getting node addresses") framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.Client)) nodeList := framework.GetReadySchedulableNodesOrDie(config.f.Client) - config.externalAddrs = framework.NodeAddresses(nodeList, api.NodeExternalIP) - if len(config.externalAddrs) < 2 { + config.ExternalAddrs = framework.NodeAddresses(nodeList, api.NodeExternalIP) + if len(config.ExternalAddrs) < 2 { // fall back to legacy IPs - config.externalAddrs = framework.NodeAddresses(nodeList, api.NodeLegacyHostIP) + config.ExternalAddrs = framework.NodeAddresses(nodeList, api.NodeLegacyHostIP) } - Expect(len(config.externalAddrs)).To(BeNumerically(">=", 2), fmt.Sprintf("At least two nodes necessary with an external or LegacyHostIP")) - config.nodes = nodeList.Items - - By("Creating the service pods in kubernetes") - podName := "netserver" - config.endpointPods = config.createNetProxyPods(podName, serviceSelector) + Expect(len(config.ExternalAddrs)).To(BeNumerically(">=", 2), fmt.Sprintf("At least two nodes necessary with an external or LegacyHostIP")) + config.Nodes = nodeList.Items By("Creating the service on top of the pods in kubernetes") - config.createNodePortService(serviceSelector) + config.createNodePortService(selector) - By("Creating test pods") - config.createTestPods() - for _, p := range config.nodePortService.Spec.Ports { + for _, p := range config.NodePortService.Spec.Ports { switch p.Protocol { case api.ProtocolUDP: - config.nodeUdpPort = int(p.NodePort) + config.NodeUdpPort = int(p.NodePort) case api.ProtocolTCP: - config.nodeHttpPort = int(p.NodePort) + config.NodeHttpPort = int(p.NodePort) default: continue } } - - epCount := len(config.endpointPods) - config.maxTries = epCount*epCount + testTries - config.clusterIP = config.nodePortService.Spec.ClusterIP - config.nodeIP = config.externalAddrs[0] + config.ClusterIP = config.NodePortService.Spec.ClusterIP + config.NodeIP = config.ExternalAddrs[0] } func (config *NetworkingTestConfig) cleanup() { @@ -437,7 +446,7 @@ func (config *NetworkingTestConfig) cleanup() { nsList, err := nsClient.List(api.ListOptions{}) if err == nil { for _, ns := range nsList.Items { - if strings.Contains(ns.Name, config.f.BaseName) && ns.Name != config.ns { + if strings.Contains(ns.Name, config.f.BaseName) && ns.Name != config.Namespace { nsClient.Delete(ns.Name) } } @@ -489,17 +498,17 @@ func (config *NetworkingTestConfig) createNetProxyPods(podName string, selector return runningPods } -func (config *NetworkingTestConfig) deleteNetProxyPod() { - pod := config.endpointPods[0] +func (config *NetworkingTestConfig) DeleteNetProxyPod() { + pod := config.EndpointPods[0] config.getPodClient().Delete(pod.Name, api.NewDeleteOptions(0)) - config.endpointPods = config.endpointPods[1:] + config.EndpointPods = config.EndpointPods[1:] // wait for pod being deleted. - err := framework.WaitForPodToDisappear(config.f.Client, config.ns, pod.Name, labels.Everything(), time.Second, wait.ForeverTestTimeout) + err := framework.WaitForPodToDisappear(config.f.Client, config.Namespace, pod.Name, labels.Everything(), time.Second, wait.ForeverTestTimeout) if err != nil { framework.Failf("Failed to delete %s pod: %v", pod.Name, err) } // wait for endpoint being removed. - err = framework.WaitForServiceEndpointsNum(config.f.Client, config.ns, nodePortServiceName, len(config.endpointPods), time.Second, wait.ForeverTestTimeout) + err = framework.WaitForServiceEndpointsNum(config.f.Client, config.Namespace, nodePortServiceName, len(config.EndpointPods), time.Second, wait.ForeverTestTimeout) if err != nil { framework.Failf("Failed to remove endpoint from service: %s", nodePortServiceName) } @@ -508,19 +517,18 @@ func (config *NetworkingTestConfig) deleteNetProxyPod() { } func (config *NetworkingTestConfig) createPod(pod *api.Pod) *api.Pod { - createdPod, err := config.getPodClient().Create(pod) - if err != nil { - framework.Failf("Failed to create %s pod: %v", pod.Name, err) - } - return createdPod + return config.getPodClient().Create(pod) } -func (config *NetworkingTestConfig) getPodClient() client.PodInterface { - return config.f.Client.Pods(config.ns) +func (config *NetworkingTestConfig) getPodClient() *framework.PodClient { + if config.podClient == nil { + config.podClient = config.f.PodClient() + } + return config.podClient } func (config *NetworkingTestConfig) getServiceClient() client.ServiceInterface { - return config.f.Client.Services(config.ns) + return config.f.Client.Services(config.Namespace) } func (config *NetworkingTestConfig) getNamespacesClient() client.NamespaceInterface {