From 585426f85f02df505222df4da69f4a4452c9e36c Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 21 Feb 2019 10:59:31 -0800 Subject: [PATCH 1/6] External connectivity test --- test/e2e/windows/hybrid_network.go | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 test/e2e/windows/hybrid_network.go diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go new file mode 100644 index 00000000000..34cbf5ec2b0 --- /dev/null +++ b/test/e2e/windows/hybrid_network.go @@ -0,0 +1,114 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package windows + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + linuxOS = "linux" + windowsOS = "windows" +) + +var ( + windowsBusyBoximage = imageutils.GetE2EImage(imageutils.TestWebserver) + linuxBusyBoxImage = "docker.io/library/nginx:1.15-alpine" +) + +var _ = SIGDescribe("Hybrid cluster network", func() { + f := framework.NewDefaultFramework("hybrid-network") + + BeforeEach(func() { + framework.SkipUnlessNodeOSDistroIs("windows") + }) + + Context("for all supported CNIs", func() { + + It("should have stable external networking for linux and windows pods", func() { + + By("creating a linux pod with external requests") + linuxPod := CreateTestPod(f, linuxBusyBoxImage, linuxOS) + cmd := []string{"/bin/sh", "-c", "nc -vz 8.8.8.8 53", "||", "nc -vz 8.8.4.4 53"} + checkExternal(f, linuxPod, linuxOS, cmd) + + By("creating a windows pod with external requests") + windowsPod := CreateTestPod(f, windowsBusyBoximage, windowsOS) + cmd = []string{"powershell", "-command", "$r=invoke-webrequest www.google.com -usebasicparsing; echo $r.StatusCode"} + out, msg := checkExternal(f, windowsPod, windowsOS, cmd) + Expect(out).To(Equal("200"), msg) + }) + + }) +}) + +func checkExternal(f *framework.Framework, podName string, os string, cmd []string]) (string, string) { + By(fmt.Sprintf("checking external connectivity of %s-container in %s pod", os, podName)) + out, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, os + "-container", cmd...) + msg := fmt.Sprintf("cmd: %v, stdout: %q, stderr: %q", cmd, out, stderr) + Expect(err).NotTo(HaveOccurred(), msg) + return out, msg +} + +func CreateTestPod(f *framework.Framework, image string, os string) string { + ns := f.Namespace.Name + cs := f.ClientSet + + podName := "pod-" + string(uuid.NewUUID()) + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: os + "-container", + Image: image, + }, + }, + NodeSelector: map[string]string{ + "beta.kubernetes.io/os": os, + }, + }, + } + if os == linuxOS { + pod.Spec.Tolerations = []v1.Toleration{ + { + Key: "key", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + } + } + _, err := cs.CoreV1().Pods(ns).Create(pod) + framework.ExpectNoError(err) + framework.ExpectNoError(f.WaitForPodRunning(podName)) + return podName +} From 59987e74108d98235d7a4e79e13c303347ce0455 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Mon, 25 Feb 2019 10:22:03 -0800 Subject: [PATCH 2/6] update bazel --- test/e2e/windows/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index 7435163c09e..f41aeafceaf 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -7,6 +7,7 @@ go_library( srcs = [ "density.go", "framework.go", + "hybrid_network.go", "networking.go", "volumes.go", ], From 2869c67076ee1d52ff2b5362f3e5bc3b24e4b07c Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Mon, 25 Feb 2019 13:14:29 -0800 Subject: [PATCH 3/6] Windows-linux connectivity --- test/e2e/windows/hybrid_network.go | 81 +++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 34cbf5ec2b0..642f5066e63 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -30,8 +30,9 @@ import ( ) const ( - linuxOS = "linux" - windowsOS = "windows" + LinuxOS = "linux" + WindowsOS = "windows" + Iterations = 5 ) var ( @@ -48,35 +49,67 @@ var _ = SIGDescribe("Hybrid cluster network", func() { Context("for all supported CNIs", func() { - It("should have stable external networking for linux and windows pods", func() { + It("should have stable networking for linux and windows pods", func() { + By("creating linux and windows pods") + linuxPod := CreateTestPod(f, linuxBusyBoxImage, LinuxOS) + windowsPod := CreateTestPod(f, windowsBusyBoximage, WindowsOS) - By("creating a linux pod with external requests") - linuxPod := CreateTestPod(f, linuxBusyBoxImage, linuxOS) - cmd := []string{"/bin/sh", "-c", "nc -vz 8.8.8.8 53", "||", "nc -vz 8.8.4.4 53"} - checkExternal(f, linuxPod, linuxOS, cmd) + By("checking connectivity to 8.8.8.8 53 (google.com) from linux") + CheckLinuxConnectivity(f, linuxPod.ObjectMeta.Name, "8.8.8.8", "53") + + By("checkin connectivity to www.google.com from windows") + CheckWindowsConnectivity(f, windowsPod.ObjectMeta.Name, "www.google.com") + + By("checking connectivity from linux to windows") + CheckLinuxConnectivity(f, linuxPod.ObjectMeta.Name, windowsPod.Status.PodIP, "80") + + By("checking connectivity from windows to linux") + CheckWindowsConnectivity(f, windowsPod.ObjectMeta.Name, linuxPod.Status.PodIP) - By("creating a windows pod with external requests") - windowsPod := CreateTestPod(f, windowsBusyBoximage, windowsOS) - cmd = []string{"powershell", "-command", "$r=invoke-webrequest www.google.com -usebasicparsing; echo $r.StatusCode"} - out, msg := checkExternal(f, windowsPod, windowsOS, cmd) - Expect(out).To(Equal("200"), msg) }) }) }) -func checkExternal(f *framework.Framework, podName string, os string, cmd []string]) (string, string) { - By(fmt.Sprintf("checking external connectivity of %s-container in %s pod", os, podName)) - out, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, os + "-container", cmd...) +func CheckContainerOutput(f *framework.Framework, podName string, os string, cmd []string) (string, string, error) { + By(fmt.Sprintf("checking connectivity of %s-container in %s", os, podName)) + out, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, os+"-container", cmd...) msg := fmt.Sprintf("cmd: %v, stdout: %q, stderr: %q", cmd, out, stderr) Expect(err).NotTo(HaveOccurred(), msg) - return out, msg + return out, msg, err } -func CreateTestPod(f *framework.Framework, image string, os string) string { - ns := f.Namespace.Name - cs := f.ClientSet +func CheckLinuxConnectivity(f *framework.Framework, podName string, address string, port string) { + successes := 0 + for i := 0; i < Iterations; i++ { + nc := fmt.Sprintf("nc -vz %s %s", address, port) + cmd := []string{"/bin/sh", "-c", nc} + _, _, err := CheckContainerOutput(f, podName, LinuxOS, cmd) + if err != nil { + break + } + successes++ + } + Expect(successes).To(Equal(Iterations)) +} +func CheckWindowsConnectivity(f *framework.Framework, podName string, address string) { + successes := 0 + for i := 0; i < Iterations; i++ { + ps := fmt.Sprintf("$r=invoke-webrequest %s -usebasicparsing; echo $r.StatusCode", address) + cmd := []string{"powershell", "-command", ps} + out, msg, err := CheckContainerOutput(f, podName, WindowsOS, cmd) + if err != nil || out != "200" { + framework.Logf(msg) + break + } + successes++ + } + Expect(successes).To(Equal(Iterations)) +} + +func CreateTestPod(f *framework.Framework, image string, os string) *v1.Pod { + containerName := fmt.Sprintf("%s-container", os) podName := "pod-" + string(uuid.NewUUID()) pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -89,8 +122,9 @@ func CreateTestPod(f *framework.Framework, image string, os string) string { Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: os + "-container", + Name: containerName, Image: image, + Ports: []v1.ContainerPort{{ContainerPort: 80}}, }, }, NodeSelector: map[string]string{ @@ -98,7 +132,7 @@ func CreateTestPod(f *framework.Framework, image string, os string) string { }, }, } - if os == linuxOS { + if os == LinuxOS { pod.Spec.Tolerations = []v1.Toleration{ { Key: "key", @@ -107,8 +141,5 @@ func CreateTestPod(f *framework.Framework, image string, os string) string { }, } } - _, err := cs.CoreV1().Pods(ns).Create(pod) - framework.ExpectNoError(err) - framework.ExpectNoError(f.WaitForPodRunning(podName)) - return podName + return f.PodClient().CreateSync(pod) } From f150874ed2686cc9e9690dadb5845ef40cb58068 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Tue, 26 Feb 2019 09:16:31 -0800 Subject: [PATCH 4/6] Requested changes and refactor --- test/e2e/windows/hybrid_network.go | 82 +++++++++++++----------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 642f5066e63..7735b8fb0da 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -30,9 +30,8 @@ import ( ) const ( - LinuxOS = "linux" - WindowsOS = "windows" - Iterations = 5 + linuxOS = "linux" + windowsOS = "windows" ) var ( @@ -49,66 +48,55 @@ var _ = SIGDescribe("Hybrid cluster network", func() { Context("for all supported CNIs", func() { - It("should have stable networking for linux and windows pods", func() { + It("should have stable networking for Linux and Windows pods", func() { By("creating linux and windows pods") - linuxPod := CreateTestPod(f, linuxBusyBoxImage, LinuxOS) - windowsPod := CreateTestPod(f, windowsBusyBoximage, WindowsOS) + linuxPod := createTestPod(f, linuxBusyBoxImage, linuxOS) + windowsPod := createTestPod(f, windowsBusyBoximage, windowsOS) - By("checking connectivity to 8.8.8.8 53 (google.com) from linux") - CheckLinuxConnectivity(f, linuxPod.ObjectMeta.Name, "8.8.8.8", "53") + By("checking connectivity to 8.8.8.8 53 (google.com) from Linux") + assertConsistentConnectivity(f, linuxPod.ObjectMeta.Name, linuxOS, linuxCheck("8.8.8.8", 53)) - By("checkin connectivity to www.google.com from windows") - CheckWindowsConnectivity(f, windowsPod.ObjectMeta.Name, "www.google.com") + By("checking connectivity to www.google.com from Windows") + assertConsistentConnectivity(f, windowsPod.ObjectMeta.Name, windowsOS, windowsCheck("www.google.com")) - By("checking connectivity from linux to windows") - CheckLinuxConnectivity(f, linuxPod.ObjectMeta.Name, windowsPod.Status.PodIP, "80") + By("checking connectivity from Linux to Windows") + assertConsistentConnectivity(f, linuxPod.ObjectMeta.Name, linuxOS, linuxCheck(windowsPod.Status.PodIP, 80)) - By("checking connectivity from windows to linux") - CheckWindowsConnectivity(f, windowsPod.ObjectMeta.Name, linuxPod.Status.PodIP) + By("checking connectivity from Windows to Linux") + assertConsistentConnectivity(f, windowsPod.ObjectMeta.Name, windowsOS, windowsCheck(linuxPod.Status.PodIP)) }) }) }) -func CheckContainerOutput(f *framework.Framework, podName string, os string, cmd []string) (string, string, error) { - By(fmt.Sprintf("checking connectivity of %s-container in %s", os, podName)) - out, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, os+"-container", cmd...) - msg := fmt.Sprintf("cmd: %v, stdout: %q, stderr: %q", cmd, out, stderr) - Expect(err).NotTo(HaveOccurred(), msg) - return out, msg, err +var ( + duration = "10s" + poll_interval = "1s" + timeout = 10 // seconds +) + +func assertConsistentConnectivity(f *framework.Framework, podName string, os string, cmd []string) { + Consistently(func() error { + By(fmt.Sprintf("checking connectivity of %s-container in %s", os, podName)) + _, _, err := f.ExecCommandInContainerWithFullOutput(podName, os+"-container", cmd...) + return err + }, duration, poll_interval).ShouldNot(HaveOccurred()) } -func CheckLinuxConnectivity(f *framework.Framework, podName string, address string, port string) { - successes := 0 - for i := 0; i < Iterations; i++ { - nc := fmt.Sprintf("nc -vz %s %s", address, port) - cmd := []string{"/bin/sh", "-c", nc} - _, _, err := CheckContainerOutput(f, podName, LinuxOS, cmd) - if err != nil { - break - } - successes++ - } - Expect(successes).To(Equal(Iterations)) +func linuxCheck(address string, port int) []string { + nc := fmt.Sprintf("nc -vz %s %v -w %v", address, port, timeout) + cmd := []string{"/bin/sh", "-c", nc} + return cmd } -func CheckWindowsConnectivity(f *framework.Framework, podName string, address string) { - successes := 0 - for i := 0; i < Iterations; i++ { - ps := fmt.Sprintf("$r=invoke-webrequest %s -usebasicparsing; echo $r.StatusCode", address) - cmd := []string{"powershell", "-command", ps} - out, msg, err := CheckContainerOutput(f, podName, WindowsOS, cmd) - if err != nil || out != "200" { - framework.Logf(msg) - break - } - successes++ - } - Expect(successes).To(Equal(Iterations)) +func windowsCheck(address string) []string { + curl := fmt.Sprintf("curl.exe %s --connect-timeout %v --fail", address, timeout) + cmd := []string{"cmd", "/c", curl} + return cmd } -func CreateTestPod(f *framework.Framework, image string, os string) *v1.Pod { +func createTestPod(f *framework.Framework, image string, os string) *v1.Pod { containerName := fmt.Sprintf("%s-container", os) podName := "pod-" + string(uuid.NewUUID()) pod := &v1.Pod{ @@ -132,7 +120,7 @@ func CreateTestPod(f *framework.Framework, image string, os string) *v1.Pod { }, }, } - if os == LinuxOS { + if os == linuxOS { pod.Spec.Tolerations = []v1.Toleration{ { Key: "key", From 25d03520298f14161b7373f97ae423a56f23e245 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Wed, 27 Feb 2019 09:47:33 -0800 Subject: [PATCH 5/6] remove key from toleration --- test/e2e/windows/hybrid_network.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 7735b8fb0da..2e87bb7b651 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -123,7 +123,6 @@ func createTestPod(f *framework.Framework, image string, os string) *v1.Pod { if os == linuxOS { pod.Spec.Tolerations = []v1.Toleration{ { - Key: "key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule, }, From a7a1f5871b2a44afc2c01f2201822536c2035cdd Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Wed, 27 Feb 2019 10:01:46 -0800 Subject: [PATCH 6/6] poll_interval to pollInterval --- test/e2e/windows/hybrid_network.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/windows/hybrid_network.go b/test/e2e/windows/hybrid_network.go index 2e87bb7b651..ba2d4a6a67f 100644 --- a/test/e2e/windows/hybrid_network.go +++ b/test/e2e/windows/hybrid_network.go @@ -71,9 +71,9 @@ var _ = SIGDescribe("Hybrid cluster network", func() { }) var ( - duration = "10s" - poll_interval = "1s" - timeout = 10 // seconds + duration = "10s" + pollInterval = "1s" + timeout = 10 // seconds ) func assertConsistentConnectivity(f *framework.Framework, podName string, os string, cmd []string) { @@ -81,7 +81,7 @@ func assertConsistentConnectivity(f *framework.Framework, podName string, os str By(fmt.Sprintf("checking connectivity of %s-container in %s", os, podName)) _, _, err := f.ExecCommandInContainerWithFullOutput(podName, os+"-container", cmd...) return err - }, duration, poll_interval).ShouldNot(HaveOccurred()) + }, duration, pollInterval).ShouldNot(HaveOccurred()) } func linuxCheck(address string, port int) []string {