From 99b1da848d49b82f4299b2b0c862d021d2b2c77a Mon Sep 17 00:00:00 2001 From: Pedro Roque Marques Date: Tue, 28 Jul 2015 11:54:32 -0700 Subject: [PATCH] Add kubelet '--resolv-conf' flag. Allow the user to specify the resolver configuration file that is used to determine the default DNS parameters. This defaults to the system's /etc/resolv.conf. --- cmd/kubelet/app/server.go | 8 ++- contrib/mesos/pkg/executor/service/service.go | 2 + hack/verify-flags/known-flags.txt | 1 + pkg/kubelet/kubelet.go | 60 ++++++++++++++----- pkg/kubelet/kubelet_test.go | 56 +++++++++++++++++ 5 files changed, 111 insertions(+), 16 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e5782a1bc13..4dce66bef46 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -123,6 +123,7 @@ type KubeletServer struct { PodCIDR string MaxPods int DockerExecHandlerName string + ResolverConfig string // Flags intended for testing @@ -253,6 +254,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.MaxPods, "max-pods", 40, "Number of Pods that can run on this Kubelet.") fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.") fs.StringVar(&s.PodCIDR, "pod-cidr", "", "The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.") + fs.StringVar(&s.ResolverConfig, "resolv-conf", kubelet.ResolvConfDefault, "Resolver configuration file used as the basis for the container DNS resolution configuration.") // Flags intended for testing, not recommended used in production environments. fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") fs.Float64Var(&s.ChaosChance, "chaos-chance", s.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]") @@ -359,6 +361,7 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { PodCIDR: s.PodCIDR, MaxPods: s.MaxPods, DockerExecHandler: dockerExecHandler, + ResolverConfig: s.ResolverConfig, }, nil } @@ -600,6 +603,7 @@ func SimpleKubelet(client *client.Client, SystemContainer: "", MaxPods: 32, DockerExecHandler: &dockertools.NativeExecHandler{}, + ResolverConfig: kubelet.ResolvConfDefault, } return &kcfg } @@ -769,6 +773,7 @@ type KubeletConfig struct { PodCIDR string MaxPods int DockerExecHandler dockertools.ExecHandler + ResolverConfig string } func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { @@ -827,7 +832,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.ConfigureCBR0, kc.PodCIDR, kc.MaxPods, - kc.DockerExecHandler) + kc.DockerExecHandler, + kc.ResolverConfig) if err != nil { return nil, nil, err diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go index b37209218d3..a026ba7314f 100644 --- a/contrib/mesos/pkg/executor/service/service.go +++ b/contrib/mesos/pkg/executor/service/service.go @@ -261,6 +261,7 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error { ConfigureCBR0: s.ConfigureCBR0, MaxPods: s.MaxPods, DockerExecHandler: dockerExecHandler, + ResolverConfig: s.ResolverConfig, } kcfg.NodeName = kcfg.Hostname @@ -362,6 +363,7 @@ func (ks *KubeletExecutorServer) createAndInitKubelet( kc.PodCIDR, kc.MaxPods, kc.DockerExecHandler, + kc.ResolverConfig, ) if err != nil { return nil, nil, err diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 7a3c37ae1e8..e2e17f9aac2 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -202,6 +202,7 @@ reject-paths repo-root report-dir required-contexts +resolv-conf resource-container resource-quota-sync-period resource-version diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 737cd829212..90254e61e0c 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -87,6 +87,9 @@ const ( // suffice because a goroutine is dedicated to check the channel and does // not block on anything else. podKillingChannelCapacity = 50 + + // system default DNS resolver configuration + ResolvConfDefault = "/etc/resolv.conf" ) var ( @@ -159,7 +162,8 @@ func NewMainKubelet( configureCBR0 bool, podCIDR string, pods int, - dockerExecHandler dockertools.ExecHandler) (*Kubelet, error) { + dockerExecHandler dockertools.ExecHandler, + resolverConfig string) (*Kubelet, error) { if rootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", rootDirectory) } @@ -275,6 +279,7 @@ func NewMainKubelet( podCIDR: podCIDR, pods: pods, syncLoopMonitor: util.AtomicValue{}, + resolverConfig: resolverConfig, } if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}); err != nil { @@ -542,6 +547,11 @@ type Kubelet struct { // Channel for sending pods to kill. podKillingCh chan *kubecontainer.Pod + + // The configuration file used as the base to generate the container's + // DNS resolver configuration file. This can be used in conjunction with + // clusterDomain and clusterDNS. + resolverConfig string } // getRootDir returns the full path to the directory under which kubelet can @@ -948,12 +958,12 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont opts.PodContainerDir = p } } - if pod.Spec.DNSPolicy == api.DNSClusterFirst { - opts.DNS, opts.DNSSearch, err = kl.getClusterDNS(pod) - if err != nil { - return nil, err - } + + opts.DNS, opts.DNSSearch, err = kl.getClusterDNS(pod) + if err != nil { + return nil, err } + return opts, nil } @@ -1086,27 +1096,47 @@ func (kl *Kubelet) podFieldSelectorRuntimeValue(fs *api.ObjectFieldSelector, pod // getClusterDNS returns a list of the DNS servers and a list of the DNS search // domains of the cluster. func (kl *Kubelet) getClusterDNS(pod *api.Pod) ([]string, []string, error) { + var hostDNS, hostSearch []string // Get host DNS settings and append them to cluster DNS settings. - f, err := os.Open("/etc/resolv.conf") - if err != nil { - return nil, nil, err - } - defer f.Close() + if kl.resolverConfig != "" { + f, err := os.Open(kl.resolverConfig) + if err != nil { + return nil, nil, err + } + defer f.Close() - hostDNS, hostSearch, err := parseResolvConf(f) - if err != nil { - return nil, nil, err + hostDNS, hostSearch, err = parseResolvConf(f) + if err != nil { + return nil, nil, err + } + } + if pod.Spec.DNSPolicy != api.DNSClusterFirst { + // When the kubelet --resolv-conf flag is set to the empty string, use + // DNS settings that override the docker default (which is to use + // /etc/resolv.conf) and effectivly disable DNS lookups. According to + // the bind documentation, the behavior of the DNS client library when + // "nameservers" are not specified is to "use the nameserver on the + // local machine". A nameserver setting of localhost is equivalent to + // this documented behavior. + if kl.resolverConfig == "" { + hostDNS = []string{"127.0.0.1"} + hostSearch = []string{"."} + } + return hostDNS, hostSearch, nil } - var dns, dnsSearch []string if kl.clusterDNS != nil { dns = append([]string{kl.clusterDNS.String()}, hostDNS...) + } else { + dns = hostDNS } if kl.clusterDomain != "" { nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, kl.clusterDomain) svcDomain := fmt.Sprintf("svc.%s", kl.clusterDomain) dnsSearch = append([]string{nsSvcDomain, svcDomain, kl.clusterDomain}, hostSearch...) + } else { + dnsSearch = hostSearch } return dns, dnsSearch, nil } diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 2ae28c5666f..6bbede5aa38 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" "os" "path" @@ -874,6 +875,61 @@ func TestParseResolvConf(t *testing.T) { } } +func TestDNSConfigurationParams(t *testing.T) { + testKubelet := newTestKubelet(t) + kubelet := testKubelet.kubelet + + clusterNS := "203.0.113.1" + kubelet.clusterDomain = "kubernetes.io" + kubelet.clusterDNS = net.ParseIP(clusterNS) + + pods := newTestPods(2) + pods[0].Spec.DNSPolicy = api.DNSClusterFirst + pods[1].Spec.DNSPolicy = api.DNSDefault + + options := make([]*kubecontainer.RunContainerOptions, 2) + for i, pod := range pods { + var err error + kubelet.volumeManager.SetVolumes(pod.UID, make(kubecontainer.VolumeMap, 0)) + options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}) + if err != nil { + t.Fatalf("failed to generate container options: %v", err) + } + } + if len(options[0].DNS) != 1 || options[0].DNS[0] != clusterNS { + t.Errorf("expected nameserver %s, got %+v", clusterNS, options[0].DNS) + } + if len(options[0].DNSSearch) == 0 || options[0].DNSSearch[0] != ".svc."+kubelet.clusterDomain { + t.Errorf("expected search %s, got %+v", ".svc."+kubelet.clusterDomain, options[0].DNSSearch) + } + if len(options[1].DNS) != 1 || options[1].DNS[0] != "127.0.0.1" { + t.Errorf("expected nameserver 127.0.0.1, got %+v", options[1].DNS) + } + if len(options[1].DNSSearch) != 1 || options[1].DNSSearch[0] != "." { + t.Errorf("expected search \".\", got %+v", options[1].DNSSearch) + } + + kubelet.resolverConfig = "/etc/resolv.conf" + for i, pod := range pods { + var err error + options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}) + if err != nil { + t.Fatalf("failed to generate container options: %v", err) + } + } + t.Logf("nameservers %+v", options[1].DNS) + if len(options[0].DNS) != len(options[1].DNS)+1 { + t.Errorf("expected prepend of cluster nameserver, got %+v", options[0].DNS) + } else if options[0].DNS[0] != clusterNS { + t.Errorf("expected nameserver %s, got %v", clusterNS, options[0].DNS[0]) + } + if len(options[0].DNSSearch) != len(options[1].DNSSearch)+3 { + t.Errorf("expected prepend of cluster domain, got %+v", options[0].DNSSearch) + } else if options[0].DNSSearch[0] != ".svc."+kubelet.clusterDomain { + t.Errorf("expected domain %s, got %s", ".svc."+kubelet.clusterDomain, options[0].DNSSearch) + } +} + type testServiceLister struct { services []api.Service }