From 834ce35ba15c60328f568149acb5d935ebed72ca Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Tue, 12 Jan 2021 14:17:43 -0800 Subject: [PATCH] directly query dns and resolve it in the e2es for windows add failure scenario for 1.1.1.1, so that no-op DNS CRI's are caught red-handed --- test/e2e/windows/dns.go | 101 ++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/test/e2e/windows/dns.go b/test/e2e/windows/dns.go index e139949c0b6..894d250a553 100644 --- a/test/e2e/windows/dns.go +++ b/test/e2e/windows/dns.go @@ -18,7 +18,6 @@ package windows import ( "context" - "regexp" "strings" v1 "k8s.io/api/core/v1" @@ -37,58 +36,100 @@ var _ = SIGDescribe("[Feature:Windows] DNS", func() { }) f := framework.NewDefaultFramework("dns") - ginkgo.It("should support configurable pod DNS servers", func() { - ginkgo.By("Preparing a test DNS service with injected DNS names...") - testInjectedIP := "1.1.1.1" - testSearchPath := "resolv.conf.local" - ginkgo.By("Creating a pod with dnsPolicy=None and customized dnsConfig...") - testUtilsPod := e2epod.NewAgnhostPod(f.Namespace.Name, "e2e-dns-utils", nil, nil, nil) - testUtilsPod.Spec.DNSPolicy = v1.DNSNone - testUtilsPod.Spec.DNSConfig = &v1.PodDNSConfig{ - Nameservers: []string{testInjectedIP}, + ginkgo.By("Getting the IP address of the internal Kubernetes service") + + svc, err := f.ClientSet.CoreV1().Services("kube-system").Get(context.TODO(), "kube-dns", metav1.GetOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Preparing a test DNS service with injected DNS names...") + // the default service IP will vary from cluster to cluster, but will always be present and is a good DNS test target + testInjectedIP := svc.Spec.ClusterIP + testSearchPath := "default.svc.cluster.local" + + ginkgo.By("Creating a windows pod with dnsPolicy=None and customized dnsConfig...") + testPod := e2epod.NewAgnhostPod(f.Namespace.Name, "e2e-dns-utils", nil, nil, nil) + testPod.Spec.DNSPolicy = v1.DNSNone + testPod.Spec.DNSConfig = &v1.PodDNSConfig{ + Nameservers: []string{testInjectedIP, "1.1.1.1"}, Searches: []string{testSearchPath}, } - testUtilsPod.Spec.NodeSelector = map[string]string{ + testPod.Spec.NodeSelector = map[string]string{ "kubernetes.io/os": "windows", } - testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), testUtilsPod, metav1.CreateOptions{}) + testPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), testPod, metav1.CreateOptions{}) framework.ExpectNoError(err) - framework.Logf("Created pod %v", testUtilsPod) + + ginkgo.By("confirming that the pod has a windows label") + framework.ExpectEqual(testPod.Spec.NodeSelector["kubernetes.io/os"], "windows") + framework.Logf("Created pod %v", testPod) defer func() { - framework.Logf("Deleting pod %s...", testUtilsPod.Name) - if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), testUtilsPod.Name, *metav1.NewDeleteOptions(0)); err != nil { - framework.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) + framework.Logf("Deleting pod %s...", testPod.Name) + if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), testPod.Name, *metav1.NewDeleteOptions(0)); err != nil { + framework.Failf("Failed to delete pod %s: %v", testPod.Name, err) } }() - framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, testUtilsPod.Name, f.Namespace.Name), "failed to wait for pod %s to be running", testUtilsPod.Name) + framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, testPod.Name, f.Namespace.Name), "failed to wait for pod %s to be running", testPod.Name) - ginkgo.By("Verifying customized DNS option is configured on pod...") + // This isn't the best 'test' but it is a great diagnostic, see later test for the 'real' test. + ginkgo.By("Calling ipconfig to get debugging info for this pod's DNS and confirm that a dns server 1.1.1.1 can be injected, along with ") cmd := []string{"ipconfig", "/all"} stdout, _, err := f.ExecWithOptions(framework.ExecOptions{ Command: cmd, Namespace: f.Namespace.Name, - PodName: testUtilsPod.Name, + PodName: testPod.Name, ContainerName: "agnhost-container", CaptureStdout: true, CaptureStderr: true, }) framework.ExpectNoError(err) - framework.Logf("ipconfig /all:\n%s", stdout) - dnsRegex, err := regexp.Compile(`DNS Servers[\s*.]*:(\s*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})+`) - framework.ExpectNoError(err) - if dnsRegex.MatchString(stdout) { - match := dnsRegex.FindString(stdout) - - if !strings.Contains(match, testInjectedIP) { - framework.Failf("customized DNS options not found in ipconfig /all, got: %s", match) - } - } else { - framework.Failf("cannot find DNS server info in ipconfig /all output: \n%s", stdout) + if !strings.Contains(stdout, "1.1.1.1") { + framework.Failf("One of the custom DNS options 1.1.1.1, not found in ipconfig /all") } + + // We've now verified that the DNS stuff is injected... now lets make sure that curl'ing 'wrong' endpoints fails, i.e. + // a negative control, to run before we run our final test... + + ginkgo.By("Verifying that curl queries FAIL for wrong URLs") + + // the below tests use curl because nslookup doesnt seem to use ndots properly + // ideally we'd use the powershell native ResolveDns but, that is not a part of agnhost images (as of k8s 1.20) + // TODO @jayunit100 add ResolveHost to agn images + + cmd = []string{"curl.exe", "-k", "https://kubernetezzzzzzzz:443"} + stdout, _, err = f.ExecWithOptions(framework.ExecOptions{ + Command: cmd, + Namespace: f.Namespace.Name, + PodName: testPod.Name, + ContainerName: "agnhost-container", + CaptureStdout: true, + CaptureStderr: true, + }) + if err == nil { + framework.Logf("Warning: Somehow the curl command succeeded... The output was \n %v", stdout) + framework.Failf("Expected a bogus URL query to fail - something is wrong with this test harness, cannot proceed.") + } + + ginkgo.By("Verifying that injected dns records for 'kubernetes' resolve to the valid ip address") + cmd = []string{"curl.exe", "-k", "https://kubernetes:443"} + stdout, _, err = f.ExecWithOptions(framework.ExecOptions{ + Command: cmd, + Namespace: f.Namespace.Name, + PodName: testPod.Name, + ContainerName: "agnhost-container", + CaptureStdout: true, + CaptureStderr: true, + }) + framework.Logf("Result of curling the kubernetes service... (Failure ok, only testing for the sake of DNS resolution) %v ... error = %v", stdout, err) + + // curl returns an error if the host isnt resolved, otherwise, it will return a passing result. + if err != nil { + framework.ExpectNoError(err) + } + // TODO: Add more test cases for other DNSPolicies. }) })