github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #81750 from aojea/fixdnsIPv6

Browse Source 
Fix e2e DNS IPv6 CI
This commit is contained in:
Kubernetes Prow Robot 2019-08-27 19:38:06 -07:00 committed by GitHub
commit 07e0cce409
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 74 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/e2e.go
View File

@ -53,6 +53,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
return nil
}, func(data []byte) {
// Run on all Ginkgo nodes
framework.SetupSuitePerGinkgoNode()
})
var _ = ginkgo.SynchronizedAfterSuite(func() {

2
test/e2e/framework/log/logger_test.go
View File

@ -84,7 +84,7 @@ func TestFailureOutput(t *testing.T) {
output: "INFO: before\nFAIL: hard-coded error\nUnexpected error:\n <*errors.errorString>: {\n s: \"an error with a long, useless description\",\n }\n an error with a long, useless description\noccurred\nINFO: after\nFAIL: true is never false either\nExpected\n <bool>: true\nto equal\n <bool>: false\n",
failure: "hard-coded error\nUnexpected error:\n <*errors.errorString>: {\n s: \"an error with a long, useless description\",\n }\n an error with a long, useless description\noccurred",
// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()
stack: "\tutil.go:1362\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1356\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
stack: "\tutil.go:1369\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1363\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
},
testResult{
name: "[Top Level] log fails",

31
test/e2e/framework/suites.go
View File

@ -116,23 +116,32 @@ func SetupSuite() {
e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
}
// Obtain the default IP family of the cluster
// Some e2e test are designed to work on IPv4 only, this global variable
// allows to adapt those tests to work on both IPv4 and IPv6
// TODO(dual-stack): dual stack clusters should pass full e2e testing at least with the primary IP family
// the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters,
// and services use the primary IP family by default
// If we´ll need to provide additional context for dual-stack, we can detect it
// because pods have two addresses (one per family)
TestContext.IPFamily = getDefaultClusterIPFamily(c)
e2elog.Logf("Cluster IP family: %s", TestContext.IPFamily)
if TestContext.NodeKiller.Enabled {
nodeKiller := NewNodeKiller(TestContext.NodeKiller, c, TestContext.Provider)
go nodeKiller.Run(TestContext.NodeKiller.NodeKillerStopCh)
}
}
// SetupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
// There are certain operations we only want to run once per overall test invocation on each Ginkgo node
// such as making some global variables accessible to all parallel executions
// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
// Ref: https://onsi.github.io/ginkgo/#parallel-specs
func SetupSuitePerGinkgoNode() {
// Obtain the default IP family of the cluster
// Some e2e test are designed to work on IPv4 only, this global variable
// allows to adapt those tests to work on both IPv4 and IPv6
// TODO: dual-stack
// the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters,
// and services use the primary IP family by default
c, err := LoadClientset()
if err != nil {
klog.Fatal("Error loading client: ", err)
}
TestContext.IPFamily = getDefaultClusterIPFamily(c)
e2elog.Logf("Cluster IP family: %s", TestContext.IPFamily)
}
// CleanupSuite is the boilerplate that can be used after tests on ginkgo were run, on the SynchronizedAfterSuite step.
// Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
// Here, the order of functions is reversed; first, the function which runs everywhere,

7
test/e2e/framework/util.go
View File

@ -415,6 +415,8 @@ func runKubernetesServiceTestContainer(c clientset.Interface, ns string) {
// getDefaultClusterIPFamily obtains the default IP family of the cluster
// using the Cluster IP address of the kubernetes service created in the default namespace
// This unequivocally identifies the default IP family because services are single family
// TODO: dual-stack may support multiple families per service
// but we can detect if a cluster is dual stack because pods have two addresses (one per family)
func getDefaultClusterIPFamily(c clientset.Interface) string {
// Get the ClusterIP of the kubernetes service created in the default namespace
svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get("kubernetes", metav1.GetOptions{})
@ -428,6 +430,11 @@ func getDefaultClusterIPFamily(c clientset.Interface) string {
return "ipv4"
}
// ClusterIsIPv6 returns true if the cluster is IPv6
func ClusterIsIPv6() bool {
return TestContext.IPFamily == "ipv6"
}
// ProviderIs returns true if the provider is included is the providers. Otherwise false.
func ProviderIs(providers ...string) bool {
for _, provider := range providers {

50
test/e2e/network/dns.go
View File

@ -49,8 +49,9 @@ var _ = SIGDescribe("DNS", func() {
namesToResolve := []string{
fmt.Sprintf("kubernetes.default.svc.%s", framework.TestContext.ClusterDNSDomain),
}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -70,8 +71,9 @@ var _ = SIGDescribe("DNS", func() {
namesToResolve = append(namesToResolve, "metadata")
}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -91,8 +93,9 @@ var _ = SIGDescribe("DNS", func() {
}
hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", dnsTestPodHostName, dnsTestServiceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
hostEntries := []string{hostFQDN, dnsTestPodHostName}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -110,8 +113,9 @@ var _ = SIGDescribe("DNS", func() {
framework.ConformanceIt("should provide /etc/hosts entries for the cluster [LinuxOnly]", func() {
hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", dnsTestPodHostName, dnsTestServiceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
hostEntries := []string{hostFQDN, dnsTestPodHostName}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostEntries, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostEntries, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -162,8 +166,9 @@ var _ = SIGDescribe("DNS", func() {
fmt.Sprintf("_http._tcp.%s.%s.svc.%s", regularService.Name, f.Namespace.Name, framework.TestContext.ClusterDNSDomain),
}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -212,8 +217,9 @@ var _ = SIGDescribe("DNS", func() {
fmt.Sprintf("_http._tcp.%s.%s.svc", regularService.Name, f.Namespace.Name),
}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, regularService.Spec.ClusterIP, "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -251,8 +257,9 @@ var _ = SIGDescribe("DNS", func() {
hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
hostNames := []string{hostFQDN, podHostname}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(nil, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(nil, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -293,8 +300,9 @@ var _ = SIGDescribe("DNS", func() {
hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
subdomain := fmt.Sprintf("%s.%s.svc.%s", serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
namesToResolve := []string{hostFQDN, subdomain}
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain)
// TODO: Validate both IPv4 and IPv6 families for dual-stack
wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, nil, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, nil, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain, framework.ClusterIsIPv6())
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")
@ -365,8 +373,14 @@ var _ = SIGDescribe("DNS", func() {
}
})
framework.ExpectNoError(err, "failed to change service type to ClusterIP for service: %s", serviceName)
wheezyProbeCmd, wheezyFileName = createTargetedProbeCommand(hostFQDN, "A", "wheezy")
jessieProbeCmd, jessieFileName = createTargetedProbeCommand(hostFQDN, "A", "jessie")
targetRecord := "A"
if framework.ClusterIsIPv6() {
targetRecord = "AAAA"
}
// TODO: For dual stack we can run from here two createTargetedProbeCommand()
// one looking for an A record and another one for an AAAA record
wheezyProbeCmd, wheezyFileName = createTargetedProbeCommand(hostFQDN, targetRecord, "wheezy")
jessieProbeCmd, jessieFileName = createTargetedProbeCommand(hostFQDN, targetRecord, "jessie")
ginkgo.By("Running these commands on wheezy: " + wheezyProbeCmd + "\n")
ginkgo.By("Running these commands on jessie: " + jessieProbeCmd + "\n")

22
test/e2e/network/dns_common.go
View File

@ -447,14 +447,18 @@ func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, servic
return dnsPod
}
func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string) (string, []string) {
func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string, isIPv6 bool) (string, []string) {
fileNames := make([]string, 0, len(namesToResolve)*2)
probeCmd := "for i in `seq 1 600`; do "
dnsRecord := "A"
if isIPv6 {
dnsRecord = "AAAA"
}
for _, name := range namesToResolve {
// Resolve by TCP and UDP DNS. Use $$(...) because $(...) is
// expanded by kubernetes (though this won't expand so should
// remain a literal, safe > sorry).
lookup := fmt.Sprintf("%s A %s AAAA", name, name)
lookup := fmt.Sprintf("%s %s", name, dnsRecord)
if strings.HasPrefix(name, "_") {
lookup = fmt.Sprintf("%s SRV", name)
}
@ -475,13 +479,16 @@ func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookup
podARecByUDPFileName := fmt.Sprintf("%s_udp@PodARecord", fileNamePrefix)
podARecByTCPFileName := fmt.Sprintf("%s_tcp@PodARecord", fileNamePrefix)
if framework.TestContext.IPFamily == "ipv6" {
probeCmd += fmt.Sprintf(`podARec=$$(getent hosts $$(hostname | awk '{print $1}') | tr ":." "-" | awk '{print $$1".%s.pod.%s"}');`, namespace, dnsDomain)
// getent doesn't work properly on Windows hosts and hostname -i doesn't return an IPv6 address
// so we have to use a different command per IP family
if isIPv6 {
probeCmd += fmt.Sprintf(`podARec=$$(getent hosts $$(hostname -s) | tr ":." "-" | awk '{print $$1".%s.pod.%s"}');`, namespace, dnsDomain)
} else {
probeCmd += fmt.Sprintf(`podARec=$$(hostname -i| awk -F. '{print $$1"-"$$2"-"$$3"-"$$4".%s.pod.%s"}');`, namespace, dnsDomain)
}
probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} A $${podARec} AAAA)" && test -n "$$check" && echo OK > /results/%s;`, podARecByUDPFileName)
probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} A $${podARec} AAAA)" && test -n "$$check" && echo OK > /results/%s;`, podARecByTCPFileName)
probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} %s)" && test -n "$$check" && echo OK > /results/%s;`, dnsRecord, podARecByUDPFileName)
probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} %s)" && test -n "$$check" && echo OK > /results/%s;`, dnsRecord, podARecByTCPFileName)
fileNames = append(fileNames, podARecByUDPFileName)
fileNames = append(fileNames, podARecByTCPFileName)
@ -506,9 +513,6 @@ func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookup
func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) {
fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve)
nameLookup := fmt.Sprintf("%s %s", nameToResolve, lookup)
if lookup == "A" {
nameLookup = fmt.Sprintf("%s A %s AAAA", nameToResolve, nameToResolve)
}
probeCmd := fmt.Sprintf("for i in `seq 1 30`; do dig +short %s > /results/%s; sleep 1; done", nameLookup, fileName)
return probeCmd, fileName
}