From daff471766359a2b2c40d973e2a39de2842a8f09 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 23 Jul 2019 09:04:47 +0200 Subject: [PATCH 1/4] e2e: Autodetect the IP family of the cluster --- test/e2e/BUILD | 1 + test/e2e/e2e.go | 23 +++++++++++++++++++++++ test/e2e/framework/test_context.go | 3 +++ 3 files changed, 27 insertions(+) diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 42b629c6f1c..0da695fe1ab 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -83,6 +83,7 @@ go_library( "//vendor/github.com/onsi/ginkgo/reporters:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 9c450f59439..0b07ff72948 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -43,6 +43,7 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/manifest" testutils "k8s.io/kubernetes/test/utils" + utilnet "k8s.io/utils/net" // ensure auth plugins are loaded _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -143,6 +144,13 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion) } + // Obtain the (primary if dual stack) ip family of the cluster + framework.TestContext.IPFamily = getClusterIPFamily(c) + e2elog.Logf("Cluster IP family: %s", framework.TestContext.IPFamily) + + // TODO(dual-stack): dual-stack clusters use two pods addresses (one per family) + // the cluster can be ipv4-ipv6 or ipv6-ipv4, order matters + // Reference common test to make the import valid. commontest.CurrentSuite = commontest.E2E @@ -276,3 +284,18 @@ func runKubernetesServiceTestContainer(c clientset.Interface, ns string) { e2elog.Logf("Output of clusterapi-tester:\n%v", logs) } } + +// getClusterIPFamily obtains the (primary) ip family of the cluster based +// on the Cluster IP address of the default kubernetes service +func getClusterIPFamily(c clientset.Interface) string { + // Get the ClusterIP of the kubernetes service created in the default namespace + svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get("kubernetes", metav1.GetOptions{}) + if err != nil { + e2elog.Failf("Failed to get kubernetes endpoints: %v", err) + } + + if utilnet.IsIPv6String(svc.Spec.ClusterIP) { + return "ipv6" + } + return "ipv4" +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 970ecfc4b3a..90a9b800112 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -164,6 +164,9 @@ type TestContextType struct { // The configuration of NodeKiller. NodeKiller NodeKillerConfig + + // The IP Family of the cluster + IPFamily string } // NodeKillerConfig describes configuration of NodeKiller -- a utility to From 0655ad338f0537257cbbd6eed8ceecfce634de42 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 23 Jul 2019 09:05:52 +0200 Subject: [PATCH 2/4] e2e: add a function to map IPv4 in IPv6 --- test/e2e/framework/BUILD | 1 + test/e2e/framework/networking_utils.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index cfbfb88ece5..83083762736 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -116,6 +116,7 @@ go_library( "//vendor/golang.org/x/net/websocket:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index b661efbae37..5544c8d8f3a 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -27,7 +27,7 @@ import ( "time" "github.com/onsi/ginkgo" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" @@ -42,6 +42,7 @@ import ( e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image" + k8utilnet "k8s.io/utils/net" ) const ( @@ -76,6 +77,16 @@ const ( var netexecImageName = imageutils.GetE2EImage(imageutils.Agnhost) +// TranslateIPv4ToIPv6 maps an IPv4 address into a valid IPv6 address +// adding the well known prefix "0::ffff:" https://tools.ietf.org/html/rfc2765 +// if the ip is IPv4 and the cluster IPFamily is IPv6, otherwise returns the same ip +func TranslateIPv4ToIPv6(ip string) string { + if TestContext.IPFamily == "ipv6" && !k8utilnet.IsIPv6String(ip) { + ip = "0::ffff:" + ip + } + return ip +} + // NewNetworkingTestConfig creates and sets up a new test config helper. func NewNetworkingTestConfig(f *Framework) *NetworkingTestConfig { config := &NetworkingTestConfig{f: f, Namespace: f.Namespace.Name, HostNetwork: true} From 54f81c3bb8abbc05dfabe6dd8d2d7a1017684e8d Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 23 Jul 2019 09:07:00 +0200 Subject: [PATCH 3/4] add ipv6 support to the hostport/HostIP e2e test --- test/e2e/scheduling/predicates.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 7414f22d205..96113ac7e9c 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -825,6 +825,7 @@ func CreateHostPortPods(f *framework.Framework, id string, replicas int, expectR // create pod which using hostport on the specified node according to the nodeSelector func createHostPortPodOnNode(f *framework.Framework, podName, ns, hostIP string, port int32, protocol v1.Protocol, nodeSelector map[string]string, expectScheduled bool) { + hostIP = framework.TranslateIPv4ToIPv6(hostIP) createPausePod(f, pausePodConfig{ Name: podName, Ports: []v1.ContainerPort{ From bd15f3f9f76855d61be1396888fae0cc6499df14 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 25 Jul 2019 13:54:37 +0200 Subject: [PATCH 4/4] Clarify IP family meaning --- test/e2e/e2e.go | 23 ++++++++++++++--------- test/e2e/framework/test_context.go | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 0b07ff72948..5bb346ff740 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -144,13 +144,17 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion) } - // Obtain the (primary if dual stack) ip family of the cluster - framework.TestContext.IPFamily = getClusterIPFamily(c) + // Obtain the default IP family of the cluster + // Some e2e test are designed to work on IPv4 only, this global variable + // allows to adapt those tests to work on both IPv4 and IPv6 + // TODO(dual-stack): dual stack clusters should pass full e2e testing at least with the primary IP family + // the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters, + // and services use the primary IP family by default + // If we´ll need to provide additional context for dual-stack, we can detect it + // because pods have two addresses (one per family) + framework.TestContext.IPFamily = getDefaultClusterIPFamily(c) e2elog.Logf("Cluster IP family: %s", framework.TestContext.IPFamily) - // TODO(dual-stack): dual-stack clusters use two pods addresses (one per family) - // the cluster can be ipv4-ipv6 or ipv6-ipv4, order matters - // Reference common test to make the import valid. commontest.CurrentSuite = commontest.E2E @@ -285,13 +289,14 @@ func runKubernetesServiceTestContainer(c clientset.Interface, ns string) { } } -// getClusterIPFamily obtains the (primary) ip family of the cluster based -// on the Cluster IP address of the default kubernetes service -func getClusterIPFamily(c clientset.Interface) string { +// getDefaultClusterIPFamily obtains the default IP family of the cluster +// using the Cluster IP address of the kubernetes service created in the default namespace +// This unequivocally identifies the default IP family because services are single family +func getDefaultClusterIPFamily(c clientset.Interface) string { // Get the ClusterIP of the kubernetes service created in the default namespace svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get("kubernetes", metav1.GetOptions{}) if err != nil { - e2elog.Failf("Failed to get kubernetes endpoints: %v", err) + e2elog.Failf("Failed to get kubernetes service ClusterIP: %v", err) } if utilnet.IsIPv6String(svc.Spec.ClusterIP) { diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 90a9b800112..35ae85c9972 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -165,7 +165,7 @@ type TestContextType struct { // The configuration of NodeKiller. NodeKiller NodeKillerConfig - // The IP Family of the cluster + // The Default IP Family of the cluster ("ipv4" or "ipv6") IPFamily string }