kubernetes/test/e2e/network/dns_common.go
Patrick Ohly 41f23f52d0 test: fix ginkgolinter issues
All of these issues were reported by https://github.com/nunnatsa/ginkgolinter.
Fixing these issues is useful (several expressions get simpler, using
framework.ExpectNoError is better because it has additional support for
failures) and a necessary step for enabling that linter in our golangci-lint
invocation.
2023-02-22 19:36:05 +01:00

546 lines
19 KiB
Go

/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package network
import (
"context"
"fmt"
"regexp"
"strings"
"time"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
dnsclient "k8s.io/kubernetes/third_party/forked/golang/net"
admissionapi "k8s.io/pod-security-admission/api"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
)
// Windows output can contain additional \r
var newLineRegexp = regexp.MustCompile("\r?\n")
type dnsTestCommon struct {
f *framework.Framework
c clientset.Interface
ns string
name string
dnsPod *v1.Pod
utilPod *v1.Pod
utilService *v1.Service
dnsServerPod *v1.Pod
cm *v1.ConfigMap
}
func newDNSTestCommon() dnsTestCommon {
framework := framework.NewDefaultFramework("dns-config-map")
framework.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
return dnsTestCommon{
f: framework,
ns: "kube-system",
}
}
func (t *dnsTestCommon) init(ctx context.Context) {
ginkgo.By("Finding a DNS pod")
label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
options := metav1.ListOptions{LabelSelector: label.String()}
namespace := "kube-system"
pods, err := t.f.ClientSet.CoreV1().Pods(namespace).List(ctx, options)
framework.ExpectNoError(err, "failed to list pods in namespace: %s", namespace)
gomega.Expect(pods.Items).ToNot(gomega.BeEmpty())
t.dnsPod = &pods.Items[0]
framework.Logf("Using DNS pod: %v", t.dnsPod.Name)
if strings.Contains(t.dnsPod.Name, "coredns") {
t.name = "coredns"
} else {
t.name = "kube-dns"
}
}
func (t *dnsTestCommon) checkDNSRecordFrom(name string, predicate func([]string) bool, target string, timeout time.Duration) {
var actual []string
err := wait.PollImmediate(
time.Duration(1)*time.Second,
timeout,
func() (bool, error) {
actual = t.runDig(name, target)
if predicate(actual) {
return true, nil
}
return false, nil
})
if err != nil {
framework.Failf("dig result did not match: %#v after %v",
actual, timeout)
}
}
// runDig queries for `dnsName`. Returns a list of responses.
func (t *dnsTestCommon) runDig(dnsName, target string) []string {
cmd := []string{"dig", "+short"}
switch target {
case "coredns":
cmd = append(cmd, "@"+t.dnsPod.Status.PodIP)
case "kube-dns":
cmd = append(cmd, "@"+t.dnsPod.Status.PodIP, "-p", "10053")
case "ptr-record":
cmd = append(cmd, "-x")
case "cluster-dns":
case "cluster-dns-ipv6":
cmd = append(cmd, "AAAA")
default:
panic(fmt.Errorf("invalid target: " + target))
}
cmd = append(cmd, dnsName)
stdout, stderr, err := e2epod.ExecWithOptions(t.f, e2epod.ExecOptions{
Command: cmd,
Namespace: t.f.Namespace.Name,
PodName: t.utilPod.Name,
ContainerName: t.utilPod.Spec.Containers[0].Name,
CaptureStdout: true,
CaptureStderr: true,
})
framework.Logf("Running dig: %v, stdout: %q, stderr: %q, err: %v",
cmd, stdout, stderr, err)
if stdout == "" {
return []string{}
}
return newLineRegexp.Split(stdout, -1)
}
func (t *dnsTestCommon) setConfigMap(ctx context.Context, cm *v1.ConfigMap) {
if t.cm != nil {
t.cm = cm
}
cm.ObjectMeta.Namespace = t.ns
cm.ObjectMeta.Name = t.name
options := metav1.ListOptions{
FieldSelector: fields.Set{
"metadata.namespace": t.ns,
"metadata.name": t.name,
}.AsSelector().String(),
}
cmList, err := t.c.CoreV1().ConfigMaps(t.ns).List(ctx, options)
framework.ExpectNoError(err, "failed to list ConfigMaps in namespace: %s", t.ns)
if len(cmList.Items) == 0 {
ginkgo.By(fmt.Sprintf("Creating the ConfigMap (%s:%s) %+v", t.ns, t.name, *cm))
_, err := t.c.CoreV1().ConfigMaps(t.ns).Create(ctx, cm, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create ConfigMap (%s:%s) %+v", t.ns, t.name, *cm)
} else {
ginkgo.By(fmt.Sprintf("Updating the ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm))
_, err := t.c.CoreV1().ConfigMaps(t.ns).Update(ctx, cm, metav1.UpdateOptions{})
framework.ExpectNoError(err, "failed to update ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm)
}
}
func (t *dnsTestCommon) fetchDNSConfigMapData(ctx context.Context) map[string]string {
if t.name == "coredns" {
pcm, err := t.c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, t.name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get DNS ConfigMap: %s", t.name)
return pcm.Data
}
return nil
}
func (t *dnsTestCommon) restoreDNSConfigMap(ctx context.Context, configMapData map[string]string) {
if t.name == "coredns" {
t.setConfigMap(ctx, &v1.ConfigMap{Data: configMapData})
t.deleteCoreDNSPods(ctx)
} else {
err := t.c.CoreV1().ConfigMaps(t.ns).Delete(ctx, t.name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
framework.Failf("Unexpected error deleting configmap %s/%s", t.ns, t.name)
}
}
}
func (t *dnsTestCommon) createUtilPodLabel(ctx context.Context, baseName string) {
// Actual port # doesn't matter, just needs to exist.
const servicePort = 10101
podName := fmt.Sprintf("%s-%s", baseName, string(uuid.NewUUID()))
ports := []v1.ContainerPort{{ContainerPort: servicePort, Protocol: v1.ProtocolTCP}}
t.utilPod = e2epod.NewAgnhostPod(t.f.Namespace.Name, podName, nil, nil, ports)
var err error
t.utilPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(ctx, t.utilPod, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod: %v", t.utilPod)
framework.Logf("Created pod %v", t.utilPod)
err = e2epod.WaitForPodNameRunningInNamespace(ctx, t.f.ClientSet, t.utilPod.Name, t.f.Namespace.Name)
framework.ExpectNoError(err, "pod failed to start running: %v", t.utilPod)
t.utilService = &v1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: t.f.Namespace.Name,
Name: baseName,
},
Spec: v1.ServiceSpec{
Selector: map[string]string{"app": baseName},
Ports: []v1.ServicePort{
{
Protocol: v1.ProtocolTCP,
Port: servicePort,
TargetPort: intstr.FromInt(servicePort),
},
},
},
}
t.utilService, err = t.c.CoreV1().Services(t.f.Namespace.Name).Create(ctx, t.utilService, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create service: %s/%s", t.f.Namespace.Name, t.utilService.ObjectMeta.Name)
framework.Logf("Created service %v", t.utilService)
}
func (t *dnsTestCommon) deleteUtilPod(ctx context.Context) {
podClient := t.c.CoreV1().Pods(t.f.Namespace.Name)
if err := podClient.Delete(ctx, t.utilPod.Name, *metav1.NewDeleteOptions(0)); err != nil {
framework.Logf("Delete of pod %v/%v failed: %v",
t.utilPod.Namespace, t.utilPod.Name, err)
}
}
// deleteCoreDNSPods manually deletes the CoreDNS pods to apply the changes to the ConfigMap.
func (t *dnsTestCommon) deleteCoreDNSPods(ctx context.Context) {
label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
options := metav1.ListOptions{LabelSelector: label.String()}
pods, err := t.f.ClientSet.CoreV1().Pods("kube-system").List(ctx, options)
framework.ExpectNoError(err, "failed to list pods of kube-system with label %q", label.String())
podClient := t.c.CoreV1().Pods(metav1.NamespaceSystem)
for _, pod := range pods.Items {
err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
framework.ExpectNoError(err, "failed to delete pod: %s", pod.Name)
}
}
func generateCoreDNSServerPod(corednsConfig *v1.ConfigMap) *v1.Pod {
podName := fmt.Sprintf("e2e-configmap-dns-server-%s", string(uuid.NewUUID()))
volumes := []v1.Volume{
{
Name: "coredns-config",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: corednsConfig.Name,
},
},
},
},
}
mounts := []v1.VolumeMount{
{
Name: "coredns-config",
MountPath: "/etc/coredns",
ReadOnly: true,
},
}
pod := e2epod.NewAgnhostPod("", podName, volumes, mounts, nil, "-conf", "/etc/coredns/Corefile")
pod.Spec.Containers[0].Command = []string{"/coredns"}
pod.Spec.DNSPolicy = "Default"
return pod
}
func generateCoreDNSConfigmap(namespaceName string, aRecords map[string]string) *v1.ConfigMap {
entries := ""
for name, ip := range aRecords {
entries += fmt.Sprintf("\n\t\t%v %v", ip, name)
}
corefileData := fmt.Sprintf(`. {
hosts {%s
}
log
}`, entries)
return &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespaceName,
GenerateName: "e2e-coredns-configmap-",
},
Data: map[string]string{
"Corefile": corefileData,
},
}
}
func (t *dnsTestCommon) createDNSPodFromObj(ctx context.Context, pod *v1.Pod) {
t.dnsServerPod = pod
var err error
t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(ctx, t.dnsServerPod, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod: %v", t.dnsServerPod)
framework.Logf("Created pod %v", t.dnsServerPod)
err = e2epod.WaitForPodNameRunningInNamespace(ctx, t.f.ClientSet, t.dnsServerPod.Name, t.f.Namespace.Name)
framework.ExpectNoError(err, "pod failed to start running: %v", t.dnsServerPod)
t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Get(ctx, t.dnsServerPod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get pod: %s", t.dnsServerPod.Name)
}
func (t *dnsTestCommon) createDNSServer(ctx context.Context, namespace string, aRecords map[string]string) {
corednsConfig := generateCoreDNSConfigmap(namespace, aRecords)
corednsConfig, err := t.c.CoreV1().ConfigMaps(namespace).Create(ctx, corednsConfig, metav1.CreateOptions{})
if err != nil {
framework.Failf("unable to create test configMap %s: %v", corednsConfig.Name, err)
}
t.createDNSPodFromObj(ctx, generateCoreDNSServerPod(corednsConfig))
}
func (t *dnsTestCommon) createDNSServerWithPtrRecord(ctx context.Context, namespace string, isIPv6 bool) {
// NOTE: PTR records are generated automatically by CoreDNS. So, if we're creating A records, we're
// going to also have PTR records. See: https://coredns.io/plugins/hosts/
var aRecords map[string]string
if isIPv6 {
aRecords = map[string]string{"my.test": "2001:db8::29"}
} else {
aRecords = map[string]string{"my.test": "192.0.2.123"}
}
t.createDNSServer(ctx, namespace, aRecords)
}
func (t *dnsTestCommon) deleteDNSServerPod(ctx context.Context) {
podClient := t.c.CoreV1().Pods(t.f.Namespace.Name)
if err := podClient.Delete(ctx, t.dnsServerPod.Name, *metav1.NewDeleteOptions(0)); err != nil {
framework.Logf("Delete of pod %v/%v failed: %v",
t.utilPod.Namespace, t.dnsServerPod.Name, err)
}
}
func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, serviceName string) *v1.Pod {
podName := "dns-test-" + string(uuid.NewUUID())
volumes := []v1.Volume{
{
Name: "results",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
}
mounts := []v1.VolumeMount{
{
Name: "results",
MountPath: "/results",
},
}
// TODO: Consider scraping logs instead of running a webserver.
dnsPod := e2epod.NewAgnhostPod(namespace, podName, volumes, mounts, nil, "test-webserver")
dnsPod.Spec.Containers[0].Name = "webserver"
querier := e2epod.NewAgnhostContainer("querier", mounts, nil, wheezyProbeCmd)
querier.Command = []string{"sh", "-c"}
jessieQuerier := v1.Container{
Name: "jessie-querier",
Image: imageutils.GetE2EImage(imageutils.JessieDnsutils),
Command: []string{"sh", "-c", jessieProbeCmd},
VolumeMounts: mounts,
}
dnsPod.Spec.Containers = append(dnsPod.Spec.Containers, querier, jessieQuerier)
dnsPod.Spec.Hostname = podHostName
dnsPod.Spec.Subdomain = serviceName
return dnsPod
}
func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string, isIPv6 bool) (string, []string) {
fileNames := make([]string, 0, len(namesToResolve)*2)
probeCmd := "for i in `seq 1 600`; do "
dnsRecord := "A"
if isIPv6 {
dnsRecord = "AAAA"
}
for _, name := range namesToResolve {
// Resolve by TCP and UDP DNS. Use $$(...) because $(...) is
// expanded by kubernetes (though this won't expand so should
// remain a literal, safe > sorry).
lookup := fmt.Sprintf("%s %s", name, dnsRecord)
if strings.HasPrefix(name, "_") {
lookup = fmt.Sprintf("%s SRV", name)
}
fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, name)
fileNames = append(fileNames, fileName)
probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s)" && test -n "$$check" && echo OK > /results/%s;`, lookup, fileName)
fileName = fmt.Sprintf("%s_tcp@%s", fileNamePrefix, name)
fileNames = append(fileNames, fileName)
probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s)" && test -n "$$check" && echo OK > /results/%s;`, lookup, fileName)
}
hostEntryCmd := `test -n "$$(getent hosts %s)" && echo OK > /results/%s;`
if framework.NodeOSDistroIs("windows") {
// We don't have getent on Windows, but we can still check the hosts file.
hostEntryCmd = `test -n "$$(grep '%s' C:/Windows/System32/drivers/etc/hosts)" && echo OK > /results/%s;`
}
for _, name := range hostEntries {
fileName := fmt.Sprintf("%s_hosts@%s", fileNamePrefix, name)
fileNames = append(fileNames, fileName)
probeCmd += fmt.Sprintf(hostEntryCmd, name, fileName)
}
if len(ptrLookupIP) > 0 {
ptrLookup, err := dnsclient.Reverseaddr(ptrLookupIP)
if err != nil {
framework.Failf("Unable to obtain reverse IP address record from IP %s: %v", ptrLookupIP, err)
}
ptrRecByUDPFileName := fmt.Sprintf("%s_udp@PTR", ptrLookupIP)
ptrRecByTCPFileName := fmt.Sprintf("%s_tcp@PTR", ptrLookupIP)
probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > /results/%s;`, ptrLookup, ptrRecByUDPFileName)
probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > /results/%s;`, ptrLookup, ptrRecByTCPFileName)
fileNames = append(fileNames, ptrRecByUDPFileName)
fileNames = append(fileNames, ptrRecByTCPFileName)
}
probeCmd += "sleep 1; done"
return probeCmd, fileNames
}
// createTargetedProbeCommand returns a command line that performs a DNS lookup for a specific record type
func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) {
fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve)
nameLookup := fmt.Sprintf("%s %s", nameToResolve, lookup)
probeCmd := fmt.Sprintf("for i in `seq 1 30`; do dig +short %s > /results/%s; sleep 1; done", nameLookup, fileName)
return probeCmd, fileName
}
func assertFilesExist(ctx context.Context, fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface) {
assertFilesContain(ctx, fileNames, fileDir, pod, client, false, "")
}
func assertFilesContain(ctx context.Context, fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface, check bool, expected string) {
var failed []string
framework.ExpectNoError(wait.PollImmediateWithContext(ctx, time.Second*5, time.Second*600, func(ctx context.Context) (bool, error) {
failed = []string{}
ctx, cancel := context.WithTimeout(ctx, framework.SingleCallTimeout)
defer cancel()
for _, fileName := range fileNames {
contents, err := client.CoreV1().RESTClient().Get().
Namespace(pod.Namespace).
Resource("pods").
SubResource("proxy").
Name(pod.Name).
Suffix(fileDir, fileName).
Do(ctx).Raw()
if err != nil {
if ctx.Err() != nil {
framework.Failf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err)
} else {
framework.Logf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err)
}
failed = append(failed, fileName)
} else if check && strings.TrimSpace(string(contents)) != expected {
framework.Logf("File %s from pod %s/%s contains '%s' instead of '%s'", fileName, pod.Namespace, pod.Name, string(contents), expected)
failed = append(failed, fileName)
}
}
if len(failed) == 0 {
return true, nil
}
framework.Logf("Lookups using %s/%s failed for: %v\n", pod.Namespace, pod.Name, failed)
return false, nil
}))
framework.ExpectEqual(len(failed), 0)
}
func validateDNSResults(ctx context.Context, f *framework.Framework, pod *v1.Pod, fileNames []string) {
ginkgo.By("submitting the pod to kubernetes")
podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name)
ginkgo.DeferCleanup(func(ctx context.Context) error {
ginkgo.By("deleting the pod")
return podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
})
if _, err := podClient.Create(ctx, pod, metav1.CreateOptions{}); err != nil {
framework.Failf("ginkgo.Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
ginkgo.By("retrieving the pod")
pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{})
if err != nil {
framework.Failf("ginkgo.Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
// Try to find results for each expected name.
ginkgo.By("looking for the results for each expected name from probers")
assertFilesExist(ctx, fileNames, "results", pod, f.ClientSet)
// TODO: probe from the host, too.
framework.Logf("DNS probes using %s/%s succeeded\n", pod.Namespace, pod.Name)
}
func validateTargetedProbeOutput(ctx context.Context, f *framework.Framework, pod *v1.Pod, fileNames []string, value string) {
ginkgo.By("submitting the pod to kubernetes")
podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name)
ginkgo.DeferCleanup(func(ctx context.Context) error {
ginkgo.By("deleting the pod")
return podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
})
if _, err := podClient.Create(ctx, pod, metav1.CreateOptions{}); err != nil {
framework.Failf("ginkgo.Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
ginkgo.By("retrieving the pod")
pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{})
if err != nil {
framework.Failf("ginkgo.Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
// Try to find the expected value for each expected name.
ginkgo.By("looking for the results for each expected name from probers")
assertFilesContain(ctx, fileNames, "results", pod, f.ClientSet, true, value)
framework.Logf("DNS probes using %s succeeded\n", pod.Name)
}