From 8858425a89195487e02743d1dec580954c877a52 Mon Sep 17 00:00:00 2001 From: Michelle Cone Date: Fri, 15 Feb 2019 12:59:50 -0500 Subject: [PATCH 1/5] Add e2e DNS tests to windows tests --- test/e2e/windows/BUILD | 1 + test/e2e/windows/dns.go | 119 +++++++ test/e2e/windows/dns_common.go | 628 +++++++++++++++++++++++++++++++++ 3 files changed, 748 insertions(+) create mode 100644 test/e2e/windows/dns.go create mode 100644 test/e2e/windows/dns_common.go diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index 349816bbbb0..b455fd619a5 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -13,6 +13,7 @@ go_library( "networking.go", "service.go", "volumes.go", + "dns.go", ], importpath = "k8s.io/kubernetes/test/e2e/windows", visibility = ["//visibility:public"], diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go new file mode 100644 index 00000000000..9ce6f0bda7c --- /dev/null +++ b/test/e2e/windows/dns.go @@ -0,0 +1,119 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package windows + +import ( + "strings" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const dnsTestPodHostName = "dns-querier-1" +const dnsTestServiceName = "dns-test-service" + +var _ = SIGDescribe("DNS", func() { + f := framework.NewDefaultFramework("dns") + + /*It("should provide DNS for pods for Hostname and Subdomain", func() { + // Create a test headless service. + By("Creating a test headless service") + testServiceSelector := map[string]string{ + "dns-test-hostname-attribute": "true", + } + serviceName := "dns-test-service-2" + podHostname := "dns-querier-2" + headlessService := framework.CreateServiceSpec(serviceName, "", true, testServiceSelector) + _, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(headlessService) + Expect(err).NotTo(HaveOccurred(), "failed to create headless service: %s", serviceName) + + defer func() { + By("deleting the test headless service") + defer GinkgoRecover() + f.ClientSet.CoreV1().Services(f.Namespace.Name).Delete(headlessService.Name, nil) + }() + + hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + hostNames := []string{hostFQDN, podHostname} + namesToResolve := []string{hostFQDN} + wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) + By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") + By("Running these commands on jessie: " + jessieProbeCmd + "\n") + + // Run a pod which probes DNS and exposes the results by HTTP. + By("creating a pod to probe DNS") + pod1 := createDNSPod(f.Namespace.Name, wheezyProbeCmd, jessieProbeCmd, dnsTestPodHostName, dnsTestServiceName) + pod1.ObjectMeta.Labels = testServiceSelector + pod1.Spec.Hostname = podHostname + pod1.Spec.Subdomain = serviceName + + validateDNSResults(f, pod1, append(wheezyFileNames, jessieFileNames...)) + })*/ + + It("should support configurable pod resolv.conf", func() { + By("Preparing a test DNS service with injected DNS names...") + testInjectedIP := "1.1.1.1" + testSearchPath := "resolv.conf.local" + + By("Creating a pod with dnsPolicy=None and customized dnsConfig...") + testUtilsPod := generateDNSUtilsPod() + testUtilsPod.Spec.DNSPolicy = v1.DNSNone + testNdotsValue := "2" + testUtilsPod.Spec.DNSConfig = &v1.PodDNSConfig{ + Nameservers: []string{testInjectedIP}, + Searches: []string{testSearchPath}, + Options: []v1.PodDNSConfigOption{ + { + Name: "ndots", + Value: &testNdotsValue, + }, + }, + } + testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(testUtilsPod) + Expect(err).NotTo(HaveOccurred(), "failed to create pod: %s", testUtilsPod.Name) + framework.Logf("Created pod %v", testUtilsPod) + defer func() { + framework.Logf("Deleting pod %s...", testUtilsPod.Name) + if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(testUtilsPod.Name, metav1.NewDeleteOptions(0)); err != nil { + framework.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) + } + }() + Expect(f.WaitForPodRunning(testUtilsPod.Name)).NotTo(HaveOccurred(), "failed to wait for pod %s to be running", testUtilsPod.Name) + + By("Verifying customized DNS option is configured on pod...") + // TODO: Figure out a better way other than checking the actual resolv,conf file. + cmd := []string{"cat", "/etc/resolv.conf"} + stdout, stderr, err := f.ExecWithOptions(framework.ExecOptions{ + Command: cmd, + Namespace: f.Namespace.Name, + PodName: testUtilsPod.Name, + ContainerName: "util", + CaptureStdout: true, + CaptureStderr: true, + }) + Expect(err).NotTo(HaveOccurred(), "failed to examine resolv,conf file on pod, stdout: %v, stderr: %v, err: %v", stdout, stderr, err) + if !strings.Contains(stdout, "ndots:2") { + framework.Failf("customized DNS options not found in resolv.conf, got: %s", stdout) + } + // TODO: Add more test cases for other DNSPolicies. + }) +}) diff --git a/test/e2e/windows/dns_common.go b/test/e2e/windows/dns_common.go new file mode 100644 index 00000000000..de0d7bca91f --- /dev/null +++ b/test/e2e/windows/dns_common.go @@ -0,0 +1,628 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package windows + +import ( + "context" + "fmt" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +type dnsTestCommon struct { + f *framework.Framework + c clientset.Interface + ns string + name string + labels []string + + dnsPod *v1.Pod + utilPod *v1.Pod + utilService *v1.Service + dnsServerPod *v1.Pod + + cm *v1.ConfigMap +} + +func newDnsTestCommon() dnsTestCommon { + return dnsTestCommon{ + f: framework.NewDefaultFramework("dns-config-map"), + ns: "kube-system", + } +} + +func (t *dnsTestCommon) init() { + By("Finding a DNS pod") + label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"})) + options := metav1.ListOptions{LabelSelector: label.String()} + + namespace := "kube-system" + pods, err := t.f.ClientSet.CoreV1().Pods(namespace).List(options) + Expect(err).NotTo(HaveOccurred(), "failed to list pods in namespace: %s", namespace) + Expect(len(pods.Items)).Should(BeNumerically(">=", 1)) + + t.dnsPod = &pods.Items[0] + framework.Logf("Using DNS pod: %v", t.dnsPod.Name) + + if strings.Contains(t.dnsPod.Name, "coredns") { + t.name = "coredns" + } else { + t.name = "kube-dns" + } +} + +func (t *dnsTestCommon) checkDNSRecord(name string, predicate func([]string) bool, timeout time.Duration) { + t.checkDNSRecordFrom(name, predicate, "kube-dns", timeout) +} + +func (t *dnsTestCommon) checkDNSRecordFrom(name string, predicate func([]string) bool, target string, timeout time.Duration) { + var actual []string + + err := wait.PollImmediate( + time.Duration(1)*time.Second, + timeout, + func() (bool, error) { + actual = t.runDig(name, target) + if predicate(actual) { + return true, nil + } + return false, nil + }) + + if err != nil { + framework.Failf("dig result did not match: %#v after %v", + actual, timeout) + } +} + +// runDig queries for `dnsName`. Returns a list of responses. +func (t *dnsTestCommon) runDig(dnsName, target string) []string { + cmd := []string{"/usr/bin/dig", "+short"} + switch target { + case "coredns": + cmd = append(cmd, "@"+t.dnsPod.Status.PodIP) + case "kube-dns": + cmd = append(cmd, "@"+t.dnsPod.Status.PodIP, "-p", "10053") + case "ptr-record": + cmd = append(cmd, "-x") + case "cluster-dns": + case "cluster-dns-ipv6": + cmd = append(cmd, "AAAA") + break + default: + panic(fmt.Errorf("invalid target: " + target)) + } + cmd = append(cmd, dnsName) + + stdout, stderr, err := t.f.ExecWithOptions(framework.ExecOptions{ + Command: cmd, + Namespace: t.f.Namespace.Name, + PodName: t.utilPod.Name, + ContainerName: "util", + CaptureStdout: true, + CaptureStderr: true, + }) + + framework.Logf("Running dig: %v, stdout: %q, stderr: %q, err: %v", + cmd, stdout, stderr, err) + + if stdout == "" { + return []string{} + } else { + return strings.Split(stdout, "\n") + } +} + +func (t *dnsTestCommon) setConfigMap(cm *v1.ConfigMap) { + if t.cm != nil { + t.cm = cm + } + + cm.ObjectMeta.Namespace = t.ns + cm.ObjectMeta.Name = t.name + + options := metav1.ListOptions{ + FieldSelector: fields.Set{ + "metadata.namespace": t.ns, + "metadata.name": t.name, + }.AsSelector().String(), + } + cmList, err := t.c.CoreV1().ConfigMaps(t.ns).List(options) + Expect(err).NotTo(HaveOccurred(), "failed to list ConfigMaps in namespace: %s", t.ns) + + if len(cmList.Items) == 0 { + By(fmt.Sprintf("Creating the ConfigMap (%s:%s) %+v", t.ns, t.name, *cm)) + _, err := t.c.CoreV1().ConfigMaps(t.ns).Create(cm) + Expect(err).NotTo(HaveOccurred(), "failed to create ConfigMap (%s:%s) %+v", t.ns, t.name, *cm) + } else { + By(fmt.Sprintf("Updating the ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm)) + _, err := t.c.CoreV1().ConfigMaps(t.ns).Update(cm) + Expect(err).NotTo(HaveOccurred(), "failed to update ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm) + } +} + +func (t *dnsTestCommon) fetchDNSConfigMapData() map[string]string { + if t.name == "coredns" { + pcm, err := t.c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(t.name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "failed to get DNS ConfigMap: %s", t.name) + return pcm.Data + } + return nil +} + +func (t *dnsTestCommon) restoreDNSConfigMap(configMapData map[string]string) { + if t.name == "coredns" { + t.setConfigMap(&v1.ConfigMap{Data: configMapData}) + t.deleteCoreDNSPods() + } else { + t.c.CoreV1().ConfigMaps(t.ns).Delete(t.name, nil) + } +} + +func (t *dnsTestCommon) deleteConfigMap() { + By(fmt.Sprintf("Deleting the ConfigMap (%s:%s)", t.ns, t.name)) + t.cm = nil + err := t.c.CoreV1().ConfigMaps(t.ns).Delete(t.name, nil) + Expect(err).NotTo(HaveOccurred(), "failed to delete config map: %s", t.name) +} + +func (t *dnsTestCommon) createUtilPodLabel(baseName string) { + // Actual port # doesn't matter, just needs to exist. + const servicePort = 10101 + + t.utilPod = &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.f.Namespace.Name, + Labels: map[string]string{"app": baseName}, + GenerateName: baseName + "-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "util", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{"sleep", "10000"}, + Ports: []v1.ContainerPort{ + {ContainerPort: servicePort, Protocol: v1.ProtocolTCP}, + }, + }, + }, + }, + } + + var err error + t.utilPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(t.utilPod) + Expect(err).NotTo(HaveOccurred(), "failed to create pod: %v", t.utilPod) + framework.Logf("Created pod %v", t.utilPod) + Expect(t.f.WaitForPodRunning(t.utilPod.Name)).NotTo(HaveOccurred(), "pod failed to start running: %v", t.utilPod) + + t.utilService = &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.f.Namespace.Name, + Name: baseName, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"app": baseName}, + Ports: []v1.ServicePort{ + { + Protocol: v1.ProtocolTCP, + Port: servicePort, + TargetPort: intstr.FromInt(servicePort), + }, + }, + }, + } + + t.utilService, err = t.c.CoreV1().Services(t.f.Namespace.Name).Create(t.utilService) + Expect(err).NotTo(HaveOccurred(), "failed to create service: %s/%s", t.f.Namespace.Name, t.utilService.ObjectMeta.Name) + framework.Logf("Created service %v", t.utilService) +} + +func (t *dnsTestCommon) deleteUtilPod() { + podClient := t.c.CoreV1().Pods(t.f.Namespace.Name) + if err := podClient.Delete(t.utilPod.Name, metav1.NewDeleteOptions(0)); err != nil { + framework.Logf("Delete of pod %v/%v failed: %v", + t.utilPod.Namespace, t.utilPod.Name, err) + } +} + +// deleteCoreDNSPods manually deletes the CoreDNS pods to apply the changes to the ConfigMap. +func (t *dnsTestCommon) deleteCoreDNSPods() { + + label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"})) + options := metav1.ListOptions{LabelSelector: label.String()} + + pods, err := t.f.ClientSet.CoreV1().Pods("kube-system").List(options) + podClient := t.c.CoreV1().Pods(metav1.NamespaceSystem) + + for _, pod := range pods.Items { + err = podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred(), "failed to delete pod: %s", pod.Name) + } +} + +func generateDNSServerPod(aRecords map[string]string) *v1.Pod { + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-dns-configmap-dns-server-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "dns", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{ + "powershell", + "Start-Sleep", + "1000", + "/usr/sbin/dnsmasq", + "-u", "root", + "-k", + "--log-facility", "-", + "-q", + }, + }, + }, + DNSPolicy: "Default", + }, + } + + for name, ip := range aRecords { + pod.Spec.Containers[0].Command = append( + pod.Spec.Containers[0].Command, + fmt.Sprintf("-A/%v/%v", name, ip)) + } + return pod +} + +func (t *dnsTestCommon) createDNSPodFromObj(pod *v1.Pod) { + t.dnsServerPod = pod + + var err error + t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(t.dnsServerPod) + Expect(err).NotTo(HaveOccurred(), "failed to create pod: %v", t.dnsServerPod) + framework.Logf("Created pod %v", t.dnsServerPod) + Expect(t.f.WaitForPodRunning(t.dnsServerPod.Name)).NotTo(HaveOccurred(), "pod failed to start running: %v", t.dnsServerPod) + + t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Get( + t.dnsServerPod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "failed to get pod: %s", t.dnsServerPod.Name) +} + +func (t *dnsTestCommon) createDNSServer(aRecords map[string]string) { + t.createDNSPodFromObj(generateDNSServerPod(aRecords)) +} + +func (t *dnsTestCommon) createDNSServerWithPtrRecord(isIPv6 bool) { + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-dns-configmap-dns-server-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "dns", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{ + "/usr/sbin/dnsmasq", + "-u", "root", + "-k", + "--log-facility", "-", + "-q", + }, + }, + }, + DNSPolicy: "Default", + }, + } + + if isIPv6 { + pod.Spec.Containers[0].Command = append( + pod.Spec.Containers[0].Command, + fmt.Sprintf("--host-record=my.test,2001:db8::29")) + } else { + pod.Spec.Containers[0].Command = append( + pod.Spec.Containers[0].Command, + fmt.Sprintf("--host-record=my.test,192.0.2.123")) + } + + t.createDNSPodFromObj(pod) +} + +func (t *dnsTestCommon) deleteDNSServerPod() { + podClient := t.c.CoreV1().Pods(t.f.Namespace.Name) + if err := podClient.Delete(t.dnsServerPod.Name, metav1.NewDeleteOptions(0)); err != nil { + framework.Logf("Delete of pod %v/%v failed: %v", + t.utilPod.Namespace, t.dnsServerPod.Name, err) + } +} + +func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, serviceName string) *v1.Pod { + dnsPod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dns-test-" + string(uuid.NewUUID()), + Namespace: namespace, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "results", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + Containers: []v1.Container{ + // TODO: Consider scraping logs instead of running a webserver. + { + Name: "webserver", + Image: imageutils.GetE2EImage(imageutils.TestWebserver), + Ports: []v1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "results", + MountPath: "C:\\results", + }, + }, + }, + { + Name: "querier", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{"cmd", "/c", wheezyProbeCmd}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "results", + MountPath: "C:\\results", + }, + }, + }, + { + Name: "jessie-querier", + Image: imageutils.GetE2EImage(imageutils.JessieDnsutils), + Command: []string{"cmd", "/c", jessieProbeCmd}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "results", + MountPath: "C:\\results", + }, + }, + }, + }, + }, + } + + dnsPod.Spec.Hostname = podHostName + dnsPod.Spec.Subdomain = serviceName + + return dnsPod +} + +func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string) (string, []string) { + fileNames := make([]string, 0, len(namesToResolve)*2) + probeCmd := "for i in `seq 1 600`; do " + for _, name := range namesToResolve { + // Resolve by TCP and UDP DNS. Use $$(...) because $(...) is + // expanded by kubernetes (though this won't expand so should + // remain a literal, safe > sorry). + lookup := "A" + if strings.HasPrefix(name, "_") { + lookup = "SRV" + } + fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, name) + fileNames = append(fileNames, fileName) + probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s %s)" && test -n "$$check" && echo OK > C:\\results\\s;`, name, lookup, fileName) + fileName = fmt.Sprintf("%s_tcp@%s", fileNamePrefix, name) + fileNames = append(fileNames, fileName) + probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s %s)" && test -n "$$check" && echo OK > C:\\results\\%s;`, name, lookup, fileName) + } + + for _, name := range hostEntries { + fileName := fmt.Sprintf("%s_hosts@%s", fileNamePrefix, name) + fileNames = append(fileNames, fileName) + probeCmd += fmt.Sprintf(`test -n "$$(getent hosts %s)" && echo OK > C:\\results\\%s;`, name, fileName) + } + + podARecByUDPFileName := fmt.Sprintf("%s_udp@PodARecord", fileNamePrefix) + podARecByTCPFileName := fmt.Sprintf("%s_tcp@PodARecord", fileNamePrefix) + probeCmd += fmt.Sprintf(`podARec=$$(hostname -i| awk -F. '{print $$1"-"$$2"-"$$3"-"$$4".%s.pod.%s"}');`, namespace, dnsDomain) + probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} A)" && test -n "$$check" && echo OK > C:\\results\\%s;`, podARecByUDPFileName) + probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} A)" && test -n "$$check" && echo OK > C:\\results\\%s;`, podARecByTCPFileName) + fileNames = append(fileNames, podARecByUDPFileName) + fileNames = append(fileNames, podARecByTCPFileName) + + if len(ptrLookupIP) > 0 { + ptrLookup := fmt.Sprintf("%s.in-addr.arpa.", strings.Join(reverseArray(strings.Split(ptrLookupIP, ".")), ".")) + ptrRecByUDPFileName := fmt.Sprintf("%s_udp@PTR", ptrLookupIP) + ptrRecByTCPFileName := fmt.Sprintf("%s_tcp@PTR", ptrLookupIP) + probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > C:\\results\\%s;`, ptrLookup, ptrRecByUDPFileName) + probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > C:\\results\\%s;`, ptrLookup, ptrRecByTCPFileName) + fileNames = append(fileNames, ptrRecByUDPFileName) + fileNames = append(fileNames, ptrRecByTCPFileName) + } + + probeCmd += "sleep 1; done" + return probeCmd, fileNames +} + +// createTargetedProbeCommand returns a command line that performs a DNS lookup for a specific record type +func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) { + fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve) + probeCmd := fmt.Sprintf("dig +short +tries=12 %s %s > C:\\results\\%s", nameToResolve, lookup, fileName) + return probeCmd, fileName +} + +func assertFilesExist(fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface) { + assertFilesContain(fileNames, fileDir, pod, client, false, "") +} + +func assertFilesContain(fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface, check bool, expected string) { + var failed []string + + framework.ExpectNoError(wait.PollImmediate(time.Second*5, time.Second*600, func() (bool, error) { + failed = []string{} + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + + for _, fileName := range fileNames { + contents, err := client.CoreV1().RESTClient().Get(). + Context(ctx). + Namespace(pod.Namespace). + Resource("pods"). + SubResource("proxy"). + Name(pod.Name). + Suffix(fileDir, fileName). + Do().Raw() + + if err != nil { + if ctx.Err() != nil { + framework.Failf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err) + } else { + framework.Logf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err) + } + failed = append(failed, fileName) + } else if check && strings.TrimSpace(string(contents)) != expected { + framework.Logf("File %s from pod %s/%s contains '%s' instead of '%s'", fileName, pod.Namespace, pod.Name, string(contents), expected) + failed = append(failed, fileName) + } + } + if len(failed) == 0 { + return true, nil + } + framework.Logf("Lookups using %s/%s failed for: %v\n", pod.Namespace, pod.Name, failed) + return false, nil + })) + Expect(len(failed)).To(Equal(0)) +} + +func validateDNSResults(f *framework.Framework, pod *v1.Pod, fileNames []string) { + By("submitting the pod to kubernetes") + podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name) + defer func() { + By("deleting the pod") + defer GinkgoRecover() + podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) + }() + if _, err := podClient.Create(pod); err != nil { + framework.Failf("Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + + By("retrieving the pod") + pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + // Try to find results for each expected name. + By("looking for the results for each expected name from probers") + assertFilesExist(fileNames, "results", pod, f.ClientSet) + + // TODO: probe from the host, too. + + framework.Logf("DNS probes using %s/%s succeeded\n", pod.Namespace, pod.Name) +} + +func validateTargetedProbeOutput(f *framework.Framework, pod *v1.Pod, fileNames []string, value string) { + By("submitting the pod to kubernetes") + podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name) + defer func() { + By("deleting the pod") + defer GinkgoRecover() + podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) + }() + if _, err := podClient.Create(pod); err != nil { + framework.Failf("Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + + By("retrieving the pod") + pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err) + } + // Try to find the expected value for each expected name. + By("looking for the results for each expected name from probers") + assertFilesContain(fileNames, "results", pod, f.ClientSet, true, value) + + framework.Logf("DNS probes using %s succeeded\n", pod.Name) +} + +func reverseArray(arr []string) []string { + for i := 0; i < len(arr)/2; i++ { + j := len(arr) - i - 1 + arr[i], arr[j] = arr[j], arr[i] + } + return arr +} + +func generateDNSUtilsPod() *v1.Pod { + return &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-dns-utils-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "util", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{"sleep", "10000"}, + }, + }, + }, + } +} From d63c90e25f808e5edaac67c9f754e30f330bb8e3 Mon Sep 17 00:00:00 2001 From: Lee Cattarin Date: Wed, 27 Feb 2019 13:44:33 -0800 Subject: [PATCH 2/5] Change custom DNS test to use ipconfig instead of resolv.conf --- test/e2e/windows/dns.go | 102 +++--- test/e2e/windows/dns_common.go | 628 --------------------------------- 2 files changed, 51 insertions(+), 679 deletions(-) delete mode 100644 test/e2e/windows/dns_common.go diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index 9ce6f0bda7c..12d220c6ab3 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,64 +17,35 @@ limitations under the License. package windows import ( + "regexp" "strings" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" ) const dnsTestPodHostName = "dns-querier-1" const dnsTestServiceName = "dns-test-service" var _ = SIGDescribe("DNS", func() { + + ginkgo.BeforeEach(func() { + framework.SkipUnlessNodeOSDistroIs("windows") + }) + f := framework.NewDefaultFramework("dns") - /*It("should provide DNS for pods for Hostname and Subdomain", func() { - // Create a test headless service. - By("Creating a test headless service") - testServiceSelector := map[string]string{ - "dns-test-hostname-attribute": "true", - } - serviceName := "dns-test-service-2" - podHostname := "dns-querier-2" - headlessService := framework.CreateServiceSpec(serviceName, "", true, testServiceSelector) - _, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(headlessService) - Expect(err).NotTo(HaveOccurred(), "failed to create headless service: %s", serviceName) - - defer func() { - By("deleting the test headless service") - defer GinkgoRecover() - f.ClientSet.CoreV1().Services(f.Namespace.Name).Delete(headlessService.Name, nil) - }() - - hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", podHostname, serviceName, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - hostNames := []string{hostFQDN, podHostname} - namesToResolve := []string{hostFQDN} - wheezyProbeCmd, wheezyFileNames := createProbeCommand(namesToResolve, hostNames, "", "wheezy", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - jessieProbeCmd, jessieFileNames := createProbeCommand(namesToResolve, hostNames, "", "jessie", f.Namespace.Name, framework.TestContext.ClusterDNSDomain) - By("Running these commands on wheezy: " + wheezyProbeCmd + "\n") - By("Running these commands on jessie: " + jessieProbeCmd + "\n") - - // Run a pod which probes DNS and exposes the results by HTTP. - By("creating a pod to probe DNS") - pod1 := createDNSPod(f.Namespace.Name, wheezyProbeCmd, jessieProbeCmd, dnsTestPodHostName, dnsTestServiceName) - pod1.ObjectMeta.Labels = testServiceSelector - pod1.Spec.Hostname = podHostname - pod1.Spec.Subdomain = serviceName - - validateDNSResults(f, pod1, append(wheezyFileNames, jessieFileNames...)) - })*/ - - It("should support configurable pod resolv.conf", func() { - By("Preparing a test DNS service with injected DNS names...") + ginkgo.It("should support configurable pod DNS servers", func() { + ginkgo.By("Preparing a test DNS service with injected DNS names...") testInjectedIP := "1.1.1.1" testSearchPath := "resolv.conf.local" - By("Creating a pod with dnsPolicy=None and customized dnsConfig...") + ginkgo.By("Creating a pod with dnsPolicy=None and customized dnsConfig...") testUtilsPod := generateDNSUtilsPod() testUtilsPod.Spec.DNSPolicy = v1.DNSNone testNdotsValue := "2" @@ -89,7 +60,7 @@ var _ = SIGDescribe("DNS", func() { }, } testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(testUtilsPod) - Expect(err).NotTo(HaveOccurred(), "failed to create pod: %s", testUtilsPod.Name) + framework.ExpectNoError(err) framework.Logf("Created pod %v", testUtilsPod) defer func() { framework.Logf("Deleting pod %s...", testUtilsPod.Name) @@ -97,12 +68,11 @@ var _ = SIGDescribe("DNS", func() { framework.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) } }() - Expect(f.WaitForPodRunning(testUtilsPod.Name)).NotTo(HaveOccurred(), "failed to wait for pod %s to be running", testUtilsPod.Name) + gomega.Expect(f.WaitForPodRunning(testUtilsPod.Name)).NotTo(gomega.HaveOccurred(), "failed to wait for pod %s to be running", testUtilsPod.Name) - By("Verifying customized DNS option is configured on pod...") - // TODO: Figure out a better way other than checking the actual resolv,conf file. - cmd := []string{"cat", "/etc/resolv.conf"} - stdout, stderr, err := f.ExecWithOptions(framework.ExecOptions{ + ginkgo.By("Verifying customized DNS option is configured on pod...") + cmd := []string{"ipconfig", "/all"} + stdout, _, err := f.ExecWithOptions(framework.ExecOptions{ Command: cmd, Namespace: f.Namespace.Name, PodName: testUtilsPod.Name, @@ -110,10 +80,40 @@ var _ = SIGDescribe("DNS", func() { CaptureStdout: true, CaptureStderr: true, }) - Expect(err).NotTo(HaveOccurred(), "failed to examine resolv,conf file on pod, stdout: %v, stderr: %v, err: %v", stdout, stderr, err) - if !strings.Contains(stdout, "ndots:2") { - framework.Failf("customized DNS options not found in resolv.conf, got: %s", stdout) + framework.ExpectNoError(err) + + framework.Logf("ipconfig /all:\n%s", stdout) + dnsRegex, err := regexp.Compile(`DNS Servers[\s*.]*:(\s*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})+`) + + if dnsRegex.MatchString(stdout) { + match := dnsRegex.FindString(stdout) + + if !strings.Contains(match, "ndots:2") { + framework.Failf("customized DNS options not found in ipconfig /all, got: %s", match) + } + } else { + framework.Failf("cannot find DNS server info in ipconfig /all output: \n%s", stdout) } // TODO: Add more test cases for other DNSPolicies. }) }) + +func generateDNSUtilsPod() *v1.Pod { + return &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-dns-utils-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "util", + Image: imageutils.GetE2EImage(imageutils.Dnsutils), + Command: []string{"sleep", "10000"}, + }, + }, + }, + } +} diff --git a/test/e2e/windows/dns_common.go b/test/e2e/windows/dns_common.go deleted file mode 100644 index de0d7bca91f..00000000000 --- a/test/e2e/windows/dns_common.go +++ /dev/null @@ -1,628 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package windows - -import ( - "context" - "fmt" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - imageutils "k8s.io/kubernetes/test/utils/image" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -type dnsTestCommon struct { - f *framework.Framework - c clientset.Interface - ns string - name string - labels []string - - dnsPod *v1.Pod - utilPod *v1.Pod - utilService *v1.Service - dnsServerPod *v1.Pod - - cm *v1.ConfigMap -} - -func newDnsTestCommon() dnsTestCommon { - return dnsTestCommon{ - f: framework.NewDefaultFramework("dns-config-map"), - ns: "kube-system", - } -} - -func (t *dnsTestCommon) init() { - By("Finding a DNS pod") - label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"})) - options := metav1.ListOptions{LabelSelector: label.String()} - - namespace := "kube-system" - pods, err := t.f.ClientSet.CoreV1().Pods(namespace).List(options) - Expect(err).NotTo(HaveOccurred(), "failed to list pods in namespace: %s", namespace) - Expect(len(pods.Items)).Should(BeNumerically(">=", 1)) - - t.dnsPod = &pods.Items[0] - framework.Logf("Using DNS pod: %v", t.dnsPod.Name) - - if strings.Contains(t.dnsPod.Name, "coredns") { - t.name = "coredns" - } else { - t.name = "kube-dns" - } -} - -func (t *dnsTestCommon) checkDNSRecord(name string, predicate func([]string) bool, timeout time.Duration) { - t.checkDNSRecordFrom(name, predicate, "kube-dns", timeout) -} - -func (t *dnsTestCommon) checkDNSRecordFrom(name string, predicate func([]string) bool, target string, timeout time.Duration) { - var actual []string - - err := wait.PollImmediate( - time.Duration(1)*time.Second, - timeout, - func() (bool, error) { - actual = t.runDig(name, target) - if predicate(actual) { - return true, nil - } - return false, nil - }) - - if err != nil { - framework.Failf("dig result did not match: %#v after %v", - actual, timeout) - } -} - -// runDig queries for `dnsName`. Returns a list of responses. -func (t *dnsTestCommon) runDig(dnsName, target string) []string { - cmd := []string{"/usr/bin/dig", "+short"} - switch target { - case "coredns": - cmd = append(cmd, "@"+t.dnsPod.Status.PodIP) - case "kube-dns": - cmd = append(cmd, "@"+t.dnsPod.Status.PodIP, "-p", "10053") - case "ptr-record": - cmd = append(cmd, "-x") - case "cluster-dns": - case "cluster-dns-ipv6": - cmd = append(cmd, "AAAA") - break - default: - panic(fmt.Errorf("invalid target: " + target)) - } - cmd = append(cmd, dnsName) - - stdout, stderr, err := t.f.ExecWithOptions(framework.ExecOptions{ - Command: cmd, - Namespace: t.f.Namespace.Name, - PodName: t.utilPod.Name, - ContainerName: "util", - CaptureStdout: true, - CaptureStderr: true, - }) - - framework.Logf("Running dig: %v, stdout: %q, stderr: %q, err: %v", - cmd, stdout, stderr, err) - - if stdout == "" { - return []string{} - } else { - return strings.Split(stdout, "\n") - } -} - -func (t *dnsTestCommon) setConfigMap(cm *v1.ConfigMap) { - if t.cm != nil { - t.cm = cm - } - - cm.ObjectMeta.Namespace = t.ns - cm.ObjectMeta.Name = t.name - - options := metav1.ListOptions{ - FieldSelector: fields.Set{ - "metadata.namespace": t.ns, - "metadata.name": t.name, - }.AsSelector().String(), - } - cmList, err := t.c.CoreV1().ConfigMaps(t.ns).List(options) - Expect(err).NotTo(HaveOccurred(), "failed to list ConfigMaps in namespace: %s", t.ns) - - if len(cmList.Items) == 0 { - By(fmt.Sprintf("Creating the ConfigMap (%s:%s) %+v", t.ns, t.name, *cm)) - _, err := t.c.CoreV1().ConfigMaps(t.ns).Create(cm) - Expect(err).NotTo(HaveOccurred(), "failed to create ConfigMap (%s:%s) %+v", t.ns, t.name, *cm) - } else { - By(fmt.Sprintf("Updating the ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm)) - _, err := t.c.CoreV1().ConfigMaps(t.ns).Update(cm) - Expect(err).NotTo(HaveOccurred(), "failed to update ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm) - } -} - -func (t *dnsTestCommon) fetchDNSConfigMapData() map[string]string { - if t.name == "coredns" { - pcm, err := t.c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(t.name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to get DNS ConfigMap: %s", t.name) - return pcm.Data - } - return nil -} - -func (t *dnsTestCommon) restoreDNSConfigMap(configMapData map[string]string) { - if t.name == "coredns" { - t.setConfigMap(&v1.ConfigMap{Data: configMapData}) - t.deleteCoreDNSPods() - } else { - t.c.CoreV1().ConfigMaps(t.ns).Delete(t.name, nil) - } -} - -func (t *dnsTestCommon) deleteConfigMap() { - By(fmt.Sprintf("Deleting the ConfigMap (%s:%s)", t.ns, t.name)) - t.cm = nil - err := t.c.CoreV1().ConfigMaps(t.ns).Delete(t.name, nil) - Expect(err).NotTo(HaveOccurred(), "failed to delete config map: %s", t.name) -} - -func (t *dnsTestCommon) createUtilPodLabel(baseName string) { - // Actual port # doesn't matter, just needs to exist. - const servicePort = 10101 - - t.utilPod = &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: t.f.Namespace.Name, - Labels: map[string]string{"app": baseName}, - GenerateName: baseName + "-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "util", - Image: imageutils.GetE2EImage(imageutils.Dnsutils), - Command: []string{"sleep", "10000"}, - Ports: []v1.ContainerPort{ - {ContainerPort: servicePort, Protocol: v1.ProtocolTCP}, - }, - }, - }, - }, - } - - var err error - t.utilPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(t.utilPod) - Expect(err).NotTo(HaveOccurred(), "failed to create pod: %v", t.utilPod) - framework.Logf("Created pod %v", t.utilPod) - Expect(t.f.WaitForPodRunning(t.utilPod.Name)).NotTo(HaveOccurred(), "pod failed to start running: %v", t.utilPod) - - t.utilService = &v1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: t.f.Namespace.Name, - Name: baseName, - }, - Spec: v1.ServiceSpec{ - Selector: map[string]string{"app": baseName}, - Ports: []v1.ServicePort{ - { - Protocol: v1.ProtocolTCP, - Port: servicePort, - TargetPort: intstr.FromInt(servicePort), - }, - }, - }, - } - - t.utilService, err = t.c.CoreV1().Services(t.f.Namespace.Name).Create(t.utilService) - Expect(err).NotTo(HaveOccurred(), "failed to create service: %s/%s", t.f.Namespace.Name, t.utilService.ObjectMeta.Name) - framework.Logf("Created service %v", t.utilService) -} - -func (t *dnsTestCommon) deleteUtilPod() { - podClient := t.c.CoreV1().Pods(t.f.Namespace.Name) - if err := podClient.Delete(t.utilPod.Name, metav1.NewDeleteOptions(0)); err != nil { - framework.Logf("Delete of pod %v/%v failed: %v", - t.utilPod.Namespace, t.utilPod.Name, err) - } -} - -// deleteCoreDNSPods manually deletes the CoreDNS pods to apply the changes to the ConfigMap. -func (t *dnsTestCommon) deleteCoreDNSPods() { - - label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"})) - options := metav1.ListOptions{LabelSelector: label.String()} - - pods, err := t.f.ClientSet.CoreV1().Pods("kube-system").List(options) - podClient := t.c.CoreV1().Pods(metav1.NamespaceSystem) - - for _, pod := range pods.Items { - err = podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) - Expect(err).NotTo(HaveOccurred(), "failed to delete pod: %s", pod.Name) - } -} - -func generateDNSServerPod(aRecords map[string]string) *v1.Pod { - pod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "e2e-dns-configmap-dns-server-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dns", - Image: imageutils.GetE2EImage(imageutils.Dnsutils), - Command: []string{ - "powershell", - "Start-Sleep", - "1000", - "/usr/sbin/dnsmasq", - "-u", "root", - "-k", - "--log-facility", "-", - "-q", - }, - }, - }, - DNSPolicy: "Default", - }, - } - - for name, ip := range aRecords { - pod.Spec.Containers[0].Command = append( - pod.Spec.Containers[0].Command, - fmt.Sprintf("-A/%v/%v", name, ip)) - } - return pod -} - -func (t *dnsTestCommon) createDNSPodFromObj(pod *v1.Pod) { - t.dnsServerPod = pod - - var err error - t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(t.dnsServerPod) - Expect(err).NotTo(HaveOccurred(), "failed to create pod: %v", t.dnsServerPod) - framework.Logf("Created pod %v", t.dnsServerPod) - Expect(t.f.WaitForPodRunning(t.dnsServerPod.Name)).NotTo(HaveOccurred(), "pod failed to start running: %v", t.dnsServerPod) - - t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Get( - t.dnsServerPod.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to get pod: %s", t.dnsServerPod.Name) -} - -func (t *dnsTestCommon) createDNSServer(aRecords map[string]string) { - t.createDNSPodFromObj(generateDNSServerPod(aRecords)) -} - -func (t *dnsTestCommon) createDNSServerWithPtrRecord(isIPv6 bool) { - pod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "e2e-dns-configmap-dns-server-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dns", - Image: imageutils.GetE2EImage(imageutils.Dnsutils), - Command: []string{ - "/usr/sbin/dnsmasq", - "-u", "root", - "-k", - "--log-facility", "-", - "-q", - }, - }, - }, - DNSPolicy: "Default", - }, - } - - if isIPv6 { - pod.Spec.Containers[0].Command = append( - pod.Spec.Containers[0].Command, - fmt.Sprintf("--host-record=my.test,2001:db8::29")) - } else { - pod.Spec.Containers[0].Command = append( - pod.Spec.Containers[0].Command, - fmt.Sprintf("--host-record=my.test,192.0.2.123")) - } - - t.createDNSPodFromObj(pod) -} - -func (t *dnsTestCommon) deleteDNSServerPod() { - podClient := t.c.CoreV1().Pods(t.f.Namespace.Name) - if err := podClient.Delete(t.dnsServerPod.Name, metav1.NewDeleteOptions(0)); err != nil { - framework.Logf("Delete of pod %v/%v failed: %v", - t.utilPod.Namespace, t.dnsServerPod.Name, err) - } -} - -func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, serviceName string) *v1.Pod { - dnsPod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dns-test-" + string(uuid.NewUUID()), - Namespace: namespace, - }, - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - Name: "results", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, - }, - Containers: []v1.Container{ - // TODO: Consider scraping logs instead of running a webserver. - { - Name: "webserver", - Image: imageutils.GetE2EImage(imageutils.TestWebserver), - Ports: []v1.ContainerPort{ - { - Name: "http", - ContainerPort: 80, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "results", - MountPath: "C:\\results", - }, - }, - }, - { - Name: "querier", - Image: imageutils.GetE2EImage(imageutils.Dnsutils), - Command: []string{"cmd", "/c", wheezyProbeCmd}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "results", - MountPath: "C:\\results", - }, - }, - }, - { - Name: "jessie-querier", - Image: imageutils.GetE2EImage(imageutils.JessieDnsutils), - Command: []string{"cmd", "/c", jessieProbeCmd}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "results", - MountPath: "C:\\results", - }, - }, - }, - }, - }, - } - - dnsPod.Spec.Hostname = podHostName - dnsPod.Spec.Subdomain = serviceName - - return dnsPod -} - -func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string) (string, []string) { - fileNames := make([]string, 0, len(namesToResolve)*2) - probeCmd := "for i in `seq 1 600`; do " - for _, name := range namesToResolve { - // Resolve by TCP and UDP DNS. Use $$(...) because $(...) is - // expanded by kubernetes (though this won't expand so should - // remain a literal, safe > sorry). - lookup := "A" - if strings.HasPrefix(name, "_") { - lookup = "SRV" - } - fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, name) - fileNames = append(fileNames, fileName) - probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s %s)" && test -n "$$check" && echo OK > C:\\results\\s;`, name, lookup, fileName) - fileName = fmt.Sprintf("%s_tcp@%s", fileNamePrefix, name) - fileNames = append(fileNames, fileName) - probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s %s)" && test -n "$$check" && echo OK > C:\\results\\%s;`, name, lookup, fileName) - } - - for _, name := range hostEntries { - fileName := fmt.Sprintf("%s_hosts@%s", fileNamePrefix, name) - fileNames = append(fileNames, fileName) - probeCmd += fmt.Sprintf(`test -n "$$(getent hosts %s)" && echo OK > C:\\results\\%s;`, name, fileName) - } - - podARecByUDPFileName := fmt.Sprintf("%s_udp@PodARecord", fileNamePrefix) - podARecByTCPFileName := fmt.Sprintf("%s_tcp@PodARecord", fileNamePrefix) - probeCmd += fmt.Sprintf(`podARec=$$(hostname -i| awk -F. '{print $$1"-"$$2"-"$$3"-"$$4".%s.pod.%s"}');`, namespace, dnsDomain) - probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search $${podARec} A)" && test -n "$$check" && echo OK > C:\\results\\%s;`, podARecByUDPFileName) - probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search $${podARec} A)" && test -n "$$check" && echo OK > C:\\results\\%s;`, podARecByTCPFileName) - fileNames = append(fileNames, podARecByUDPFileName) - fileNames = append(fileNames, podARecByTCPFileName) - - if len(ptrLookupIP) > 0 { - ptrLookup := fmt.Sprintf("%s.in-addr.arpa.", strings.Join(reverseArray(strings.Split(ptrLookupIP, ".")), ".")) - ptrRecByUDPFileName := fmt.Sprintf("%s_udp@PTR", ptrLookupIP) - ptrRecByTCPFileName := fmt.Sprintf("%s_tcp@PTR", ptrLookupIP) - probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > C:\\results\\%s;`, ptrLookup, ptrRecByUDPFileName) - probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > C:\\results\\%s;`, ptrLookup, ptrRecByTCPFileName) - fileNames = append(fileNames, ptrRecByUDPFileName) - fileNames = append(fileNames, ptrRecByTCPFileName) - } - - probeCmd += "sleep 1; done" - return probeCmd, fileNames -} - -// createTargetedProbeCommand returns a command line that performs a DNS lookup for a specific record type -func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) { - fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve) - probeCmd := fmt.Sprintf("dig +short +tries=12 %s %s > C:\\results\\%s", nameToResolve, lookup, fileName) - return probeCmd, fileName -} - -func assertFilesExist(fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface) { - assertFilesContain(fileNames, fileDir, pod, client, false, "") -} - -func assertFilesContain(fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface, check bool, expected string) { - var failed []string - - framework.ExpectNoError(wait.PollImmediate(time.Second*5, time.Second*600, func() (bool, error) { - failed = []string{} - - ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) - defer cancel() - - for _, fileName := range fileNames { - contents, err := client.CoreV1().RESTClient().Get(). - Context(ctx). - Namespace(pod.Namespace). - Resource("pods"). - SubResource("proxy"). - Name(pod.Name). - Suffix(fileDir, fileName). - Do().Raw() - - if err != nil { - if ctx.Err() != nil { - framework.Failf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err) - } else { - framework.Logf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err) - } - failed = append(failed, fileName) - } else if check && strings.TrimSpace(string(contents)) != expected { - framework.Logf("File %s from pod %s/%s contains '%s' instead of '%s'", fileName, pod.Namespace, pod.Name, string(contents), expected) - failed = append(failed, fileName) - } - } - if len(failed) == 0 { - return true, nil - } - framework.Logf("Lookups using %s/%s failed for: %v\n", pod.Namespace, pod.Name, failed) - return false, nil - })) - Expect(len(failed)).To(Equal(0)) -} - -func validateDNSResults(f *framework.Framework, pod *v1.Pod, fileNames []string) { - By("submitting the pod to kubernetes") - podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name) - defer func() { - By("deleting the pod") - defer GinkgoRecover() - podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) - }() - if _, err := podClient.Create(pod); err != nil { - framework.Failf("Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err) - } - - framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) - - By("retrieving the pod") - pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) - if err != nil { - framework.Failf("Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err) - } - // Try to find results for each expected name. - By("looking for the results for each expected name from probers") - assertFilesExist(fileNames, "results", pod, f.ClientSet) - - // TODO: probe from the host, too. - - framework.Logf("DNS probes using %s/%s succeeded\n", pod.Namespace, pod.Name) -} - -func validateTargetedProbeOutput(f *framework.Framework, pod *v1.Pod, fileNames []string, value string) { - By("submitting the pod to kubernetes") - podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name) - defer func() { - By("deleting the pod") - defer GinkgoRecover() - podClient.Delete(pod.Name, metav1.NewDeleteOptions(0)) - }() - if _, err := podClient.Create(pod); err != nil { - framework.Failf("Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err) - } - - framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) - - By("retrieving the pod") - pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) - if err != nil { - framework.Failf("Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err) - } - // Try to find the expected value for each expected name. - By("looking for the results for each expected name from probers") - assertFilesContain(fileNames, "results", pod, f.ClientSet, true, value) - - framework.Logf("DNS probes using %s succeeded\n", pod.Name) -} - -func reverseArray(arr []string) []string { - for i := 0; i < len(arr)/2; i++ { - j := len(arr) - i - 1 - arr[i], arr[j] = arr[j], arr[i] - } - return arr -} - -func generateDNSUtilsPod() *v1.Pod { - return &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "e2e-dns-utils-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "util", - Image: imageutils.GetE2EImage(imageutils.Dnsutils), - Command: []string{"sleep", "10000"}, - }, - }, - }, - } -} From c0e079bb15fb33eb0f50e6b2fb3810082bd11b17 Mon Sep 17 00:00:00 2001 From: Lee Cattarin Date: Mon, 4 Mar 2019 15:44:57 -0800 Subject: [PATCH 3/5] Run hack/update-bazel --- test/e2e/windows/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index b455fd619a5..64487332f04 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -6,6 +6,7 @@ go_library( name = "go_default_library", srcs = [ "density.go", + "dns.go", "framework.go", "gmsa.go", "hybrid_network.go", @@ -13,7 +14,6 @@ go_library( "networking.go", "service.go", "volumes.go", - "dns.go", ], importpath = "k8s.io/kubernetes/test/e2e/windows", visibility = ["//visibility:public"], From 9e0a1d20c4d8f92282ee31ac87c655b5010ee0fc Mon Sep 17 00:00:00 2001 From: Lee Cattarin Date: Thu, 7 Mar 2019 11:48:06 -0800 Subject: [PATCH 4/5] Remove ndots check as it doesn't apply to windows --- test/e2e/windows/dns.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index 12d220c6ab3..04f075df7ca 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -48,16 +48,9 @@ var _ = SIGDescribe("DNS", func() { ginkgo.By("Creating a pod with dnsPolicy=None and customized dnsConfig...") testUtilsPod := generateDNSUtilsPod() testUtilsPod.Spec.DNSPolicy = v1.DNSNone - testNdotsValue := "2" testUtilsPod.Spec.DNSConfig = &v1.PodDNSConfig{ Nameservers: []string{testInjectedIP}, Searches: []string{testSearchPath}, - Options: []v1.PodDNSConfigOption{ - { - Name: "ndots", - Value: &testNdotsValue, - }, - }, } testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(testUtilsPod) framework.ExpectNoError(err) @@ -88,7 +81,7 @@ var _ = SIGDescribe("DNS", func() { if dnsRegex.MatchString(stdout) { match := dnsRegex.FindString(stdout) - if !strings.Contains(match, "ndots:2") { + if !strings.Contains(match, testInjectedIP) { framework.Failf("customized DNS options not found in ipconfig /all, got: %s", match) } } else { From 0fd18fc06f0cf8d436e6dc120b9d7489f8ece176 Mon Sep 17 00:00:00 2001 From: Patrick Lang Date: Fri, 23 Aug 2019 23:47:20 +0000 Subject: [PATCH 5/5] Updating log after framework code refactor --- test/e2e/windows/dns.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index 04f075df7ca..add791f2f39 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -23,10 +23,10 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" imageutils "k8s.io/kubernetes/test/utils/image" "github.com/onsi/ginkgo" - "github.com/onsi/gomega" ) const dnsTestPodHostName = "dns-querier-1" @@ -54,14 +54,14 @@ var _ = SIGDescribe("DNS", func() { } testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(testUtilsPod) framework.ExpectNoError(err) - framework.Logf("Created pod %v", testUtilsPod) + e2elog.Logf("Created pod %v", testUtilsPod) defer func() { - framework.Logf("Deleting pod %s...", testUtilsPod.Name) + e2elog.Logf("Deleting pod %s...", testUtilsPod.Name) if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(testUtilsPod.Name, metav1.NewDeleteOptions(0)); err != nil { - framework.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) + e2elog.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) } }() - gomega.Expect(f.WaitForPodRunning(testUtilsPod.Name)).NotTo(gomega.HaveOccurred(), "failed to wait for pod %s to be running", testUtilsPod.Name) + framework.ExpectNoError(f.WaitForPodRunning(testUtilsPod.Name), "failed to wait for pod %s to be running", testUtilsPod.Name) ginkgo.By("Verifying customized DNS option is configured on pod...") cmd := []string{"ipconfig", "/all"} @@ -75,17 +75,17 @@ var _ = SIGDescribe("DNS", func() { }) framework.ExpectNoError(err) - framework.Logf("ipconfig /all:\n%s", stdout) + e2elog.Logf("ipconfig /all:\n%s", stdout) dnsRegex, err := regexp.Compile(`DNS Servers[\s*.]*:(\s*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})+`) if dnsRegex.MatchString(stdout) { match := dnsRegex.FindString(stdout) if !strings.Contains(match, testInjectedIP) { - framework.Failf("customized DNS options not found in ipconfig /all, got: %s", match) + e2elog.Failf("customized DNS options not found in ipconfig /all, got: %s", match) } } else { - framework.Failf("cannot find DNS server info in ipconfig /all output: \n%s", stdout) + e2elog.Failf("cannot find DNS server info in ipconfig /all output: \n%s", stdout) } // TODO: Add more test cases for other DNSPolicies. })