From 789920955a9cda382d1d4d5557469c881d8c93f7 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Thu, 6 Jun 2019 14:43:13 -0700 Subject: [PATCH 1/2] add e2e for dualstack add test for network connectivity check update tests complete tests Review feedback run update-bazel feature flag connectivity test remove dead code add todo --- test/e2e/network/BUILD | 3 + test/e2e/network/dual_stack.go | 299 +++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 test/e2e/network/dual_stack.go diff --git a/test/e2e/network/BUILD b/test/e2e/network/BUILD index 1f815577642..bd7f95c10d0 100644 --- a/test/e2e/network/BUILD +++ b/test/e2e/network/BUILD @@ -13,6 +13,7 @@ go_library( "dns_configmap.go", "dns_scale_records.go", "doc.go", + "dual_stack.go", "example_cluster_dns.go", "firewall.go", "framework.go", @@ -34,6 +35,7 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/controller/endpoint:go_default_library", "//pkg/master/ports:go_default_library", + "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/networking/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", @@ -60,6 +62,7 @@ go_library( "//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/auth:go_default_library", + "//test/e2e/framework/deployment:go_default_library", "//test/e2e/framework/endpoints:go_default_library", "//test/e2e/framework/ingress:go_default_library", "//test/e2e/framework/log:go_default_library", diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go new file mode 100644 index 00000000000..64b4dc42ae2 --- /dev/null +++ b/test/e2e/network/dual_stack.go @@ -0,0 +1,299 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + "net" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2edeploy "k8s.io/kubernetes/test/e2e/framework/deployment" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +// Tests for ipv6 dual stack feature +var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { + f := framework.NewDefaultFramework("dualstack") + + var cs clientset.Interface + var podClient *framework.PodClient + + ginkgo.BeforeEach(func() { + cs = f.ClientSet + podClient = f.PodClient() + }) + + ginkgo.It("should have ipv4 and ipv6 internal node ip", func() { + // TODO (aramase) can switch to new function to get all nodes + nodeList := framework.GetReadySchedulableNodesOrDie(cs) + + for _, node := range nodeList.Items { + // get all internal ips for node + internalIPs := e2enode.GetAddresses(&node, v1.NodeInternalIP) + + gomega.Expect(len(internalIPs)).Should(gomega.BeNumerically("==", 2)) + // assert 2 ips belong to different families + gomega.Expect(isIPv4(internalIPs[0]) != isIPv4(internalIPs[1])).Should(gomega.BeTrue()) + } + }) + + ginkgo.It("should have ipv4 and ipv6 node podCIDRs", func() { + // TODO (aramase) can switch to new function to get all nodes + nodeList := framework.GetReadySchedulableNodesOrDie(cs) + + for _, node := range nodeList.Items { + gomega.Expect(len(node.Spec.PodCIDRs)).Should(gomega.BeNumerically("==", 2)) + // assert podCIDR is same as podCIDRs[0] + gomega.Expect(node.Spec.PodCIDR).Should(gomega.BeEquivalentTo(node.Spec.PodCIDRs[0])) + // assert one is ipv4 and other is ipv6 + gomega.Expect(isIPv4CIDR(node.Spec.PodCIDRs[0]) != isIPv4CIDR(node.Spec.PodCIDRs[1])).Should(gomega.BeTrue()) + } + }) + + ginkgo.It("should create pod, add ipv6 and ipv4 ip to pod ips", func() { + podName := "pod-dualstack-ips" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Labels: map[string]string{"test": "dualstack-pod-ips"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "dualstack-pod-ips", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + }, + }, + }, + } + + ginkgo.By("submitting the pod to kubernetes") + podClient.CreateSync(pod) + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + + p, err := podClient.Get(pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Failed to get pod %q", pod.Name) + + gomega.Expect(p.Status.PodIP).ShouldNot(gomega.BeEquivalentTo("")) + gomega.Expect(p.Status.PodIPs).ShouldNot(gomega.BeNil()) + + // validate there are 2 ips in podIPs + gomega.Expect(len(p.Status.PodIPs)).Should(gomega.BeNumerically("==", 2)) + // validate first ip in PodIPs is same as PodIP + gomega.Expect(p.Status.PodIPs[0].IP).Should(gomega.BeEquivalentTo(p.Status.PodIP)) + // assert 2 pod ips belong to different families + gomega.Expect(isIPv4(p.Status.PodIPs[0].IP) != isIPv4(p.Status.PodIPs[1].IP)).Should(gomega.BeTrue()) + + ginkgo.By("deleting the pod") + err = podClient.Delete(pod.Name, metav1.NewDeleteOptions(30)) + framework.ExpectNoError(err, "failed to delete pod") + }) + + // takes close to 140s to complete, so doesn't need to be marked [SLOW] + // this test is tagged with phase2 so we can skip this until phase 2 is completed and merged + // TODO (aramase) remove phase 2 tag once phase 2 of dual stack is merged + ginkgo.It("should be able to reach pod on ipv4 and ipv6 ip [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { + serverDeploymentName := "dualstack-server" + clientDeploymentName := "dualstack-client" + + // get all schedulable nodes to determine the number of replicas for pods + // this is to ensure connectivity from all nodes on cluster + nodeList := framework.GetReadySchedulableNodesOrDie(cs) + gomega.Expect(nodeList).NotTo(gomega.BeNil()) + + if len(nodeList.Items) < 1 { + e2elog.Failf("Expect at least 1 node, got %v", len(nodeList.Items)) + } + + replicas := int32(len(nodeList.Items)) + + // to ensure all the pods land on different nodes and we can thereby + // validate connectivity across all nodes. + serverDeploymentSpec := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: serverDeploymentName, + Labels: map[string]string{"test": "dualstack-test-server"}, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"test": "dualstack-test-server"}}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"test": "dualstack-test-server"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "dualstack-test-server", + Image: imageutils.GetE2EImage(imageutils.TestWebserver), + }, + }, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "test", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"dualstack-test-server"}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + }, + }, + } + + clientDeploymentSpec := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientDeploymentName, + Labels: map[string]string{"test": "dualstack-test-client"}, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"test": "dualstack-test-client"}}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"test": "dualstack-test-client"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "dualstack-test-client", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Command: []string{"sleep", "3600"}, + }, + }, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "test", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"dualstack-test-client"}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + }, + }, + } + + serverDeployment, err := cs.AppsV1().Deployments(f.Namespace.Name).Create(serverDeploymentSpec) + framework.ExpectNoError(err) + + clientDeployment, err := cs.AppsV1().Deployments(f.Namespace.Name).Create(clientDeploymentSpec) + framework.ExpectNoError(err) + + err = e2edeploy.WaitForDeploymentComplete(cs, serverDeployment) + framework.ExpectNoError(err) + err = e2edeploy.WaitForDeploymentComplete(cs, clientDeployment) + framework.ExpectNoError(err) + + serverPods, err := e2edeploy.GetPodsForDeployment(cs, serverDeployment) + framework.ExpectNoError(err) + + clientPods, err := e2edeploy.GetPodsForDeployment(cs, clientDeployment) + framework.ExpectNoError(err) + + assertNetworkConnectivity(f, *serverPods, *clientPods) + }) +}) + +var ( + duration = "10s" + pollInterval = "1s" + timeout = 10 +) + +func assertNetworkConnectivity(f *framework.Framework, serverPods v1.PodList, clientPods v1.PodList) { + // curl from each client pod to all server pods to assert connectivity + var serverIPs []string + for _, pod := range serverPods.Items { + if pod.Status.PodIPs == nil || len(pod.Status.PodIPs) != 2 { + e2elog.Failf("PodIPs list not expected value, got %v", pod.Status.PodIPs) + } + if isIPv4(pod.Status.PodIPs[0].IP) == isIPv4(pod.Status.PodIPs[1].IP) { + e2elog.Failf("PodIPs should belong to different families, got %v", pod.Status.PodIPs) + } + serverIPs = append(serverIPs, pod.Status.PodIPs[0].IP, pod.Status.PodIPs[1].IP) + } + + for _, clientPod := range clientPods.Items { + for _, ip := range serverIPs { + gomega.Consistently(func() error { + ginkgo.By(fmt.Sprintf("checking connectivity from pod %s to server ip %s", clientPod.Name, ip)) + cmd := checkNetworkConnectivity(ip, "80") + _, _, err := f.ExecCommandInContainerWithFullOutput(clientPod.Name, "dualstack-test-client", cmd...) + return err + }, duration, pollInterval).ShouldNot(gomega.HaveOccurred()) + } + } +} + +func checkNetworkConnectivity(ip, port string) []string { + curl := fmt.Sprintf("curl -g --connect-timeout %v http://%s", 10, net.JoinHostPort(ip, port)) + cmd := []string{"/bin/sh", "-c", curl} + return cmd +} + +// isIPv4 checks if the provided ip belongs to ipv4 family. +// If ip belongs to ipv4 family, return true else it returns false +// TODO (aramase) move this to https://github.com/kubernetes/utils/blob/master/net/net.go +func isIPv4(ip string) bool { + return net.ParseIP(ip).To4() != nil +} + +// isIPv4CIDR checks if the provided cidr block belongs to ipv4 family. +// If cidr belongs to ipv4 family, return true else it returns false +// TODO (aramase) move this to https://github.com/kubernetes/utils/blob/master/net/net.go +func isIPv4CIDR(cidr string) bool { + ip, _, err := net.ParseCIDR(cidr) + framework.ExpectNoError(err) + return isIPv4(ip.String()) +} From bc13a39ccf7301db4e6ea38fa2be1d4a366b4f4d Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Mon, 8 Jul 2019 00:24:28 -0700 Subject: [PATCH 2/2] use framework.ExpectEqual for tests --- test/e2e/network/dual_stack.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go index 64b4dc42ae2..085f2676782 100644 --- a/test/e2e/network/dual_stack.go +++ b/test/e2e/network/dual_stack.go @@ -54,9 +54,9 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { // get all internal ips for node internalIPs := e2enode.GetAddresses(&node, v1.NodeInternalIP) - gomega.Expect(len(internalIPs)).Should(gomega.BeNumerically("==", 2)) + framework.ExpectEqual(len(internalIPs), 2) // assert 2 ips belong to different families - gomega.Expect(isIPv4(internalIPs[0]) != isIPv4(internalIPs[1])).Should(gomega.BeTrue()) + framework.ExpectEqual(isIPv4(internalIPs[0]) != isIPv4(internalIPs[1]), true) } }) @@ -65,11 +65,11 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { nodeList := framework.GetReadySchedulableNodesOrDie(cs) for _, node := range nodeList.Items { - gomega.Expect(len(node.Spec.PodCIDRs)).Should(gomega.BeNumerically("==", 2)) + framework.ExpectEqual(len(node.Spec.PodCIDRs), 2) // assert podCIDR is same as podCIDRs[0] - gomega.Expect(node.Spec.PodCIDR).Should(gomega.BeEquivalentTo(node.Spec.PodCIDRs[0])) + framework.ExpectEqual(node.Spec.PodCIDR, node.Spec.PodCIDRs[0]) // assert one is ipv4 and other is ipv6 - gomega.Expect(isIPv4CIDR(node.Spec.PodCIDRs[0]) != isIPv4CIDR(node.Spec.PodCIDRs[1])).Should(gomega.BeTrue()) + framework.ExpectEqual(isIPv4CIDR(node.Spec.PodCIDRs[0]) != isIPv4CIDR(node.Spec.PodCIDRs[1]), true) } }) @@ -102,11 +102,11 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { gomega.Expect(p.Status.PodIPs).ShouldNot(gomega.BeNil()) // validate there are 2 ips in podIPs - gomega.Expect(len(p.Status.PodIPs)).Should(gomega.BeNumerically("==", 2)) + framework.ExpectEqual(len(p.Status.PodIPs), 2) // validate first ip in PodIPs is same as PodIP - gomega.Expect(p.Status.PodIPs[0].IP).Should(gomega.BeEquivalentTo(p.Status.PodIP)) + framework.ExpectEqual(p.Status.PodIP, p.Status.PodIPs[0].IP) // assert 2 pod ips belong to different families - gomega.Expect(isIPv4(p.Status.PodIPs[0].IP) != isIPv4(p.Status.PodIPs[1].IP)).Should(gomega.BeTrue()) + framework.ExpectEqual(isIPv4(p.Status.PodIPs[0].IP) != isIPv4(p.Status.PodIPs[1].IP), true) ginkgo.By("deleting the pod") err = podClient.Delete(pod.Name, metav1.NewDeleteOptions(30))