From 766d79bbf5d7a547698d0732f1a0d30b2454e730 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 27 Aug 2019 12:54:45 +0200 Subject: [PATCH 1/2] Make TestContext.IPFamily global for parallel testing it turns out that the framework.TestContext.IPFamily variable is not available for the DNS tests if they don't run in the initial Ginkgo node when running in parallel. We add a function to the framework to allow us to run command only once per each Ginkgo node parallel execution. It also adds a method to detect if the cluster is IPv6. The use of the framework.TestContext.IPFamily variable guarantees consistency all over the testing because this variable is only assigned at the beginning of the testing. --- test/e2e/e2e.go | 1 + test/e2e/framework/log/logger_test.go | 2 +- test/e2e/framework/suites.go | 31 +++++++++++++++++---------- test/e2e/framework/util.go | 7 ++++++ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 38062834aa4..e771187313d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -53,6 +53,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { return nil }, func(data []byte) { // Run on all Ginkgo nodes + framework.SetupSuitePerGinkgoNode() }) var _ = ginkgo.SynchronizedAfterSuite(func() { diff --git a/test/e2e/framework/log/logger_test.go b/test/e2e/framework/log/logger_test.go index 76439c06298..8a51ff51cfa 100644 --- a/test/e2e/framework/log/logger_test.go +++ b/test/e2e/framework/log/logger_test.go @@ -84,7 +84,7 @@ func TestFailureOutput(t *testing.T) { output: "INFO: before\nFAIL: hard-coded error\nUnexpected error:\n <*errors.errorString>: {\n s: \"an error with a long, useless description\",\n }\n an error with a long, useless description\noccurred\nINFO: after\nFAIL: true is never false either\nExpected\n : true\nto equal\n : false\n", failure: "hard-coded error\nUnexpected error:\n <*errors.errorString>: {\n s: \"an error with a long, useless description\",\n }\n an error with a long, useless description\noccurred", // TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4() - stack: "\tutil.go:1362\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1356\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n", + stack: "\tutil.go:1369\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1363\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n", }, testResult{ name: "[Top Level] log fails", diff --git a/test/e2e/framework/suites.go b/test/e2e/framework/suites.go index 823d8273336..9dedeca4747 100644 --- a/test/e2e/framework/suites.go +++ b/test/e2e/framework/suites.go @@ -116,23 +116,32 @@ func SetupSuite() { e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion) } - // Obtain the default IP family of the cluster - // Some e2e test are designed to work on IPv4 only, this global variable - // allows to adapt those tests to work on both IPv4 and IPv6 - // TODO(dual-stack): dual stack clusters should pass full e2e testing at least with the primary IP family - // the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters, - // and services use the primary IP family by default - // If we´ll need to provide additional context for dual-stack, we can detect it - // because pods have two addresses (one per family) - TestContext.IPFamily = getDefaultClusterIPFamily(c) - e2elog.Logf("Cluster IP family: %s", TestContext.IPFamily) - if TestContext.NodeKiller.Enabled { nodeKiller := NewNodeKiller(TestContext.NodeKiller, c, TestContext.Provider) go nodeKiller.Run(TestContext.NodeKiller.NodeKillerStopCh) } } +// SetupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step. +// There are certain operations we only want to run once per overall test invocation on each Ginkgo node +// such as making some global variables accessible to all parallel executions +// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite +// Ref: https://onsi.github.io/ginkgo/#parallel-specs +func SetupSuitePerGinkgoNode() { + // Obtain the default IP family of the cluster + // Some e2e test are designed to work on IPv4 only, this global variable + // allows to adapt those tests to work on both IPv4 and IPv6 + // TODO: dual-stack + // the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters, + // and services use the primary IP family by default + c, err := LoadClientset() + if err != nil { + klog.Fatal("Error loading client: ", err) + } + TestContext.IPFamily = getDefaultClusterIPFamily(c) + e2elog.Logf("Cluster IP family: %s", TestContext.IPFamily) +} + // CleanupSuite is the boilerplate that can be used after tests on ginkgo were run, on the SynchronizedAfterSuite step. // Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). // Here, the order of functions is reversed; first, the function which runs everywhere, diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 49b6ee13abd..daf25b8f27b 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -415,6 +415,8 @@ func runKubernetesServiceTestContainer(c clientset.Interface, ns string) { // getDefaultClusterIPFamily obtains the default IP family of the cluster // using the Cluster IP address of the kubernetes service created in the default namespace // This unequivocally identifies the default IP family because services are single family +// TODO: dual-stack may support multiple families per service +// but we can detect if a cluster is dual stack because pods have two addresses (one per family) func getDefaultClusterIPFamily(c clientset.Interface) string { // Get the ClusterIP of the kubernetes service created in the default namespace svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get("kubernetes", metav1.GetOptions{}) @@ -428,6 +430,11 @@ func getDefaultClusterIPFamily(c clientset.Interface) string { return "ipv4" } +// ClusterIsIPv6 returns true if the cluster is IPv6 +func ClusterIsIPv6() bool { + return TestContext.IPFamily == "ipv6" +} + // ProviderIs returns true if the provider is included is the providers. Otherwise false. func ProviderIs(providers ...string) bool { for _, provider := range providers { From 930a1321dac4a8178184e8c64c1ab56057ef1862 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 27 Aug 2019 12:57:40 +0200 Subject: [PATCH 2/2] DNS e2e tests differentiate between IP family The e2e DNS tests differentiate between IP family to avoid false positives with dual stack clusters. --- test/e2e/network/dns.go | 50 ++++++++++++++++++++++------------ test/e2e/network/dns_common.go | 22 +++++++++------ 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/test/e2e/network/dns.go b/test/e2e/network/dns.go index 778af0f76af..0fe340c7d72 100644 --- a/test/e2e/network/dns.go +++ b/test/e2e/network/dns.go @@ -49,8 +49,9 @@ var _ = SIGDescribe("DNS", func() { namesToResolve := []string{ fmt.Sprintf("kubernetes.default.svc.%s", framework.TestContext.ClusterDNSDomain), } - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -70,8 +71,9 @@ var _ = SIGDescribe("DNS", func() { namesToResolve = append(namesToResolve, "metadata") } - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -91,8 +93,9 @@ var _ = SIGDescribe("DNS", func() { } hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", dnsTestPodHostName, dnsTestServiceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) hostEntries := []string{hostFQDN, dnsTestPodHostName} - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -110,8 +113,9 @@ var _ = SIGDescribe("DNS", func() { framework.ConformanceIt("should provide /etc/hosts entries for the cluster [LinuxOnly]", func() { hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", dnsTestPodHostName, dnsTestServiceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) hostEntries := []string{hostFQDN, dnsTestPodHostName} - wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -162,8 +166,9 @@ var _ = SIGDescribe("DNS", func() { fmt.Sprintf("_http._tcp.%s.%s.svc.%s", regularService.Name, f.Namespace.Name, framework.TestContext.ClusterDNSDomain), } - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -212,8 +217,9 @@ var _ = SIGDescribe("DNS", func() { fmt.Sprintf("_http._tcp.%s.%s.svc", regularService.Name, f.Namespace.Name), } - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -251,8 +257,9 @@ var _ = SIGDescribe("DNS", func() { hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) hostNames := []string{hostFQDN, podHostname} - wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -293,8 +300,9 @@ var _ = SIGDescribe("DNS", func() { hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) subdomain := fmt.Sprintf("%s.%s.svc.%s", serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) namesToResolve := []string{hostFQDN, subdomain} - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + // TODO: Validate both IPv4 and IPv6 families for dual-stack + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6()) ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") @@ -365,8 +373,14 @@ var _ = SIGDescribe("DNS", func() { } }) framework.ExpectNoError(err, "failed to change service type to ClusterIP for service: %s", serviceName) - wheezyProbeCmd, wheezyFileName = createTargetedProbeCommand(hostFQDN, "A", "wheezy") - jessieProbeCmd, jessieFileName = createTargetedProbeCommand(hostFQDN, "A", "jessie") + targetRecord := "A" + if framework.ClusterIsIPv6() { + targetRecord = "AAAA" + } + // TODO: For dual stack we can run from here two createTargetedProbeCommand() + // one looking for an A record and another one for an AAAA record + wheezyProbeCmd, wheezyFileName = createTargetedProbeCommand(hostFQDN, targetRecord, "wheezy") + jessieProbeCmd, jessieFileName = createTargetedProbeCommand(hostFQDN, targetRecord, "jessie") ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n") diff --git a/test/e2e/network/dns_common.go b/test/e2e/network/dns_common.go index 44aabc20061..bbd2e416730 100644 --- a/test/e2e/network/dns_common.go +++ b/test/e2e/network/dns_common.go @@ -447,14 +447,18 @@ func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, servic return dnsPod } -func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string) (string, []string) { +func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string, isIPv6 bool) (string, []string) { fileNames := make([]string, 0, len(namesToResolve)*2) probeCmd := "for i in `seq 1 600`; do " + dnsRecord := "A" + if isIPv6 { + dnsRecord = "AAAA" + } for _, name := range namesToResolve { // Resolve by TCP and UDP DNS. Use $$(...) because $(...) is // expanded by kubernetes (though this won't expand so should // remain a literal, safe > sorry). - lookup := fmt.Sprintf("%s A %s AAAA", name, name) + lookup := fmt.Sprintf("%s %s", name, dnsRecord) if strings.HasPrefix(name, "_") { lookup = fmt.Sprintf("%s SRV", name) } @@ -475,13 +479,16 @@ func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookup podARecByUDPFileName := fmt.Sprintf("%s_udp@PodARecord", fileNamePrefix) podARecByTCPFileName := fmt.Sprintf("%s_tcp@PodARecord", fileNamePrefix) - if framework.TestContext.IPFamily == "ipv6" { - probeCmd += fmt.Sprintf(`podARec=$$(getent hosts $$(hostname | awk '{print $1}') | tr ":." "-" | awk '{print $$1".%s.pod.%s"}');`, namespace, dnsDomain) + // getent doesn't work properly on Windows hosts and hostname -i doesn't return an IPv6 address + // so we have to use a different command per IP family + if isIPv6 { + probeCmd += fmt.Sprintf(`podARec=$$(getent hosts $$(hostname -s) | tr ":." "-" | awk '{print $$1".%s.pod.%s"}');`, namespace, dnsDomain) } else { probeCmd += fmt.Sprintf(`podARec=$$(hostname -i| awk -F. '{print $$1"-"$$2"-"$$3"-"$$4".%s.pod.%s"}');`, namespace, dnsDomain) } - probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} A $${podARec} AAAA)" && test -n "$$check" && echo OK > /results/%s;`, podARecByUDPFileName) - probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} A $${podARec} AAAA)" && test -n "$$check" && echo OK > /results/%s;`, podARecByTCPFileName) + + probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} %s)" && test -n "$$check" && echo OK > /results/%s;`, dnsRecord, podARecByUDPFileName) + probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} %s)" && test -n "$$check" && echo OK > /results/%s;`, dnsRecord, podARecByTCPFileName) fileNames = append(fileNames, podARecByUDPFileName) fileNames = append(fileNames, podARecByTCPFileName) @@ -506,9 +513,6 @@ func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookup func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) { fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve) nameLookup := fmt.Sprintf("%s %s", nameToResolve, lookup) - if lookup == "A" { - nameLookup = fmt.Sprintf("%s A %s AAAA", nameToResolve, nameToResolve) - } probeCmd := fmt.Sprintf("for i in `seq 1 30`; do dig +short %s > /results/%s; sleep 1; done", nameLookup, fileName) return probeCmd, fileName }