Add kubelet DNS flags & api disable for DNS

This adds --cluster_dns and --cluster_domain flags to kubelet.  If
non-empty, kubelet will set docker --dns and --dns-search flags based on
these.  It uses the cluster DNS and appends the hosts's DNS servers.
Likewise for DNS search domains.

This also adds API support to bypass cluster DNS entirely, needed to
bootstrap DNS.

cmd/kubelet/kubelet.go

@@ -63,12 +63,15 @@ var (
 	cAdvisorPort            = flag.Uint("cadvisor_port", 4194, "The port of the localhost cAdvisor endpoint")
 	oomScoreAdj             = flag.Int("oom_score_adj", -900, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
 	apiServerList           util.StringList
+	clusterDomain           = flag.String("cluster_domain", "", "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
+	clusterDNS              = util.IP(nil)
 )
 
 func init() {
 	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
 	flag.Var(&address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
 	flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
+	flag.Var(&clusterDNS, "cluster_dns", "IP address for a cluster DNS server.  If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
 }
 
 func setupRunOnce() {
@@ -114,6 +117,8 @@ func main() {
 		RegistryBurst:           *registryBurst,
 		MinimumGCAge:            *minimumGCAge,
 		MaxContainerCount:       *maxContainerCount,
+		ClusterDomain:           *clusterDomain,
+		ClusterDNS:              clusterDNS,
 		Runonce:                 *runonce,
 		Port:                    *port,
 		CAdvisorPort:            *cAdvisorPort,

pkg/api/types.go

@@ -448,11 +448,27 @@ type PodList struct {
 	Items []Pod `json:"items"`
 }
 
+// DNSPolicy defines how a pod's DNS will be configured.
+type DNSPolicy string
+
+const (
+	// DNSClusterFirst indicates that the pod should use cluster DNS
+	// first, if it is available, then fall back on the default (as
+	// determined by kubelet) DNS settings.
+	DNSClusterFirst DNSPolicy = "ClusterFirst"
+
+	// DNSDefault indicates that the pod should use the default (as
+	// determined by kubelet) DNS settings.
+	DNSDefault DNSPolicy = "Default"
+)
+
 // PodSpec is a description of a pod
 type PodSpec struct {
 	Volumes       []Volume      `json:"volumes"`
 	Containers    []Container   `json:"containers"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 
@@ -1029,6 +1045,8 @@ type ContainerManifest struct {
 	Volumes       []Volume      `json:"volumes"`
 	Containers    []Container   `json:"containers"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy"`
 }
 
 // ContainerManifestList is used to communicate container manifests to kubelet.

pkg/api/v1beta1/conversion.go

@@ -411,6 +411,7 @@ func init() {
 			if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
 				return err
 			}
+			out.DNSPolicy = DNSPolicy(in.DNSPolicy)
 			out.Version = "v1beta2"
 			return nil
 		},
@@ -424,6 +425,7 @@ func init() {
 			if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
 				return err
 			}
+			out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
 			return nil
 		},
 

pkg/api/v1beta1/types.go

@@ -60,6 +60,8 @@ type ContainerManifest struct {
 	Volumes       []Volume      `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
 	Containers    []Container   `json:"containers" description:"list of containers belonging to the pod"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
 }
 
 // ContainerManifestList is used to communicate container manifests to kubelet.
@@ -827,11 +829,27 @@ type EventList struct {
 
 // Backported from v1beta3 to replace ContainerManifest
 
+// DNSPolicy defines how a pod's DNS will be configured.
+type DNSPolicy string
+
+const (
+	// DNSClusterFirst indicates that the pod should use cluster DNS
+	// first, if it is available, then fall back on the default (as
+	// determined by kubelet) DNS settings.
+	DNSClusterFirst DNSPolicy = "ClusterFirst"
+
+	// DNSDefault indicates that the pod should use the default (as
+	// determined by kubelet) DNS settings.
+	DNSDefault DNSPolicy = "Default"
+)
+
 // PodSpec is a description of a pod
 type PodSpec struct {
 	Volumes       []Volume      `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
 	Containers    []Container   `json:"containers" description:"list of containers belonging to the pod"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node
 	NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`
 

pkg/api/v1beta2/conversion.go

@@ -286,6 +286,7 @@ func init() {
 			if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
 				return err
 			}
+			out.DNSPolicy = DNSPolicy(in.DNSPolicy)
 			out.Version = "v1beta2"
 			return nil
 		},
@@ -299,6 +300,7 @@ func init() {
 			if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
 				return err
 			}
+			out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
 			return nil
 		},
 

pkg/api/v1beta2/types.go

@@ -817,6 +817,8 @@ type ContainerManifest struct {
 	Volumes       []Volume      `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
 	Containers    []Container   `json:"containers" description:"list of containers belonging to the pod"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
 }
 
 // ContainerManifestList is used to communicate container manifests to kubelet.
@@ -828,11 +830,27 @@ type ContainerManifestList struct {
 
 // Backported from v1beta3 to replace ContainerManifest
 
+// DNSPolicy defines how a pod's DNS will be configured.
+type DNSPolicy string
+
+const (
+	// DNSClusterFirst indicates that the pod should use cluster DNS
+	// first, if it is available, then fall back on the default (as
+	// determined by kubelet) DNS settings.
+	DNSClusterFirst DNSPolicy = "ClusterFirst"
+
+	// DNSDefault indicates that the pod should use the default (as
+	// determined by kubelet) DNS settings.
+	DNSDefault DNSPolicy = "Default"
+)
+
 // PodSpec is a description of a pod
 type PodSpec struct {
 	Volumes       []Volume      `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
 	Containers    []Container   `json:"containers" description:"list of containers belonging to the pod"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node
 	NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`
 

pkg/api/v1beta3/types.go

@@ -457,11 +457,27 @@ type RestartPolicy struct {
 	Never     *RestartPolicyNever     `json:"never,omitempty"`
 }
 
+// DNSPolicy defines how a pod's DNS will be configured.
+type DNSPolicy string
+
+const (
+	// DNSClusterFirst indicates that the pod should use cluster DNS
+	// first, if it is available, then fall back on the default (as
+	// determined by kubelet) DNS settings.
+	DNSClusterFirst DNSPolicy = "ClusterFirst"
+
+	// DNSDefault indicates that the pod should use the default (as
+	// determined by kubelet) DNS settings.
+	DNSDefault DNSPolicy = "Default"
+)
+
 // PodSpec is a description of a pod
 type PodSpec struct {
 	Volumes       []Volume      `json:"volumes"`
 	Containers    []Container   `json:"containers"`
 	RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
+	// Optional: Set DNS policy.  Defaults to "ClusterFirst"
+	DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 

pkg/api/validation/validation.go

@@ -326,6 +326,7 @@ func ValidateManifest(manifest *api.ContainerManifest) errs.ValidationErrorList
 	allErrs = append(allErrs, vErrs.Prefix("volumes")...)
 	allErrs = append(allErrs, validateContainers(manifest.Containers, allVolumes).Prefix("containers")...)
 	allErrs = append(allErrs, validateRestartPolicy(&manifest.RestartPolicy).Prefix("restartPolicy")...)
+	allErrs = append(allErrs, validateDNSPolicy(&manifest.DNSPolicy).Prefix("dnsPolicy")...)
 	return allErrs
 }
 
@@ -350,6 +351,20 @@ func validateRestartPolicy(restartPolicy *api.RestartPolicy) errs.ValidationErro
 	return allErrors
 }
 
+func validateDNSPolicy(dnsPolicy *api.DNSPolicy) errs.ValidationErrorList {
+	allErrors := errs.ValidationErrorList{}
+	switch *dnsPolicy {
+	case "":
+		// TODO: move this out to standard defaulting logic, when that is ready.
+		*dnsPolicy = api.DNSClusterFirst // Default value.
+	case api.DNSClusterFirst, api.DNSDefault:
+		break
+	default:
+		allErrors = append(allErrors, errs.NewFieldNotSupported("", dnsPolicy))
+	}
+	return allErrors
+}
+
 // ValidatePod tests if required fields in the pod are set.
 func ValidatePod(pod *api.Pod) errs.ValidationErrorList {
 	allErrs := errs.ValidationErrorList{}
@@ -375,6 +390,7 @@ func ValidatePodSpec(spec *api.PodSpec) errs.ValidationErrorList {
 	allErrs = append(allErrs, vErrs.Prefix("volumes")...)
 	allErrs = append(allErrs, validateContainers(spec.Containers, allVolumes).Prefix("containers")...)
 	allErrs = append(allErrs, validateRestartPolicy(&spec.RestartPolicy).Prefix("restartPolicy")...)
+	allErrs = append(allErrs, validateDNSPolicy(&spec.DNSPolicy).Prefix("dnsPolicy")...)
 	allErrs = append(allErrs, validateLabels(spec.NodeSelector, "nodeSelector")...)
 	return allErrs
 }

pkg/kubelet/kubelet.go

@@ -19,6 +19,8 @@ package kubelet
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"path"
@@ -69,7 +71,9 @@ func NewMainKubelet(
 	pullBurst int,
 	minimumGCAge time.Duration,
 	maxContainerCount int,
-	sourcesReady SourcesReadyFn) *Kubelet {
+	sourcesReady SourcesReadyFn,
+	clusterDomain string,
+	clusterDNS net.IP) *Kubelet {
 	return &Kubelet{
 		hostname:              hn,
 		dockerClient:          dc,
@@ -86,6 +90,8 @@ func NewMainKubelet(
 		minimumGCAge:          minimumGCAge,
 		maxContainerCount:     maxContainerCount,
 		sourcesReady:          sourcesReady,
+		clusterDomain:         clusterDomain,
+		clusterDNS:            clusterDNS,
 	}
 }
 
@@ -138,6 +144,12 @@ type Kubelet struct {
 	// Optional, minimum age required for garbage collection.  If zero, no limit.
 	minimumGCAge      time.Duration
 	maxContainerCount int
+
+	// If non-empty, use this for container DNS search.
+	clusterDomain string
+
+	// If non-nil, use this for container DNS server.
+	clusterDNS net.IP
 }
 
 // GetRootDir returns the full path to the directory under which kubelet can
@@ -561,12 +573,18 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
 	} else if container.Privileged {
 		return "", fmt.Errorf("container requested privileged mode, but it is disallowed globally.")
 	}
-	err = kl.dockerClient.StartContainer(dockerContainer.ID, &docker.HostConfig{
+	hc := &docker.HostConfig{
 		PortBindings: portBindings,
 		Binds:        binds,
 		NetworkMode:  netMode,
 		Privileged:   privileged,
-	})
+	}
+	if pod.Spec.DNSPolicy == api.DNSClusterFirst {
+		if err := kl.applyClusterDNS(hc, pod); err != nil {
+			return "", err
+		}
+	}
+	err = kl.dockerClient.StartContainer(dockerContainer.ID, hc)
 	if err != nil {
 		if ref != nil {
 			record.Eventf(ref, "failed", "failed",
@@ -588,6 +606,62 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
 	return dockertools.DockerID(dockerContainer.ID), err
 }
 
+func (kl *Kubelet) applyClusterDNS(hc *docker.HostConfig, pod *api.BoundPod) error {
+	// Get host DNS settings and append them to cluster DNS settings.
+	f, err := os.Open("/etc/resolv.conf")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	hostDNS, hostSearch, err := parseResolvConf(f)
+	if err != nil {
+		return err
+	}
+
+	if kl.clusterDNS != nil {
+		hc.DNS = append([]string{kl.clusterDNS.String()}, hostDNS...)
+	}
+	if kl.clusterDomain != "" {
+		nsDomain := fmt.Sprintf("%s.%s", pod.Namespace, kl.clusterDomain)
+		hc.DNSSearch = append([]string{nsDomain, kl.clusterDomain}, hostSearch...)
+	}
+	return nil
+}
+
+// Returns the list of DNS servers and DNS search domains.
+func parseResolvConf(reader io.Reader) (nameservers []string, searches []string, err error) {
+	file, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Lines of the form "nameserver 1.2.3.4" accumulate.
+	nameservers = []string{}
+
+	// Lines of the form "search example.com" overrule - last one wins.
+	searches = []string{}
+
+	lines := strings.Split(string(file), "\n")
+	for l := range lines {
+		trimmed := strings.TrimSpace(lines[l])
+		if strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		fields := strings.Fields(trimmed)
+		if len(fields) == 0 {
+			continue
+		}
+		if fields[0] == "nameserver" {
+			nameservers = append(nameservers, fields[1:]...)
+		}
+		if fields[0] == "search" {
+			searches = fields[1:]
+		}
+	}
+	return nameservers, searches, nil
+}
+
 // Kill a docker container
 func (kl *Kubelet) killContainer(dockerContainer *docker.APIContainers) error {
 	return kl.killContainerByID(dockerContainer.ID, dockerContainer.Names[0])

pkg/kubelet/kubelet_test.go

@@ -1723,3 +1723,47 @@ func TestGarbageCollectImages(t *testing.T) {
 		t.Errorf("unexpected images removed: %v", fakeDocker.RemovedImages)
 	}
 }
+
+func TestParseResolvConf(t *testing.T) {
+	testCases := []struct {
+		data        string
+		nameservers []string
+		searches    []string
+	}{
+		{"", []string{}, []string{}},
+		{" ", []string{}, []string{}},
+		{"\n", []string{}, []string{}},
+		{"\t\n\t", []string{}, []string{}},
+		{"#comment\n", []string{}, []string{}},
+		{" #comment\n", []string{}, []string{}},
+		{"#comment\n#comment", []string{}, []string{}},
+		{"#comment\nnameserver", []string{}, []string{}},
+		{"#comment\nnameserver\nsearch", []string{}, []string{}},
+		{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}},
+		{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}},
+		{"search foo", []string{}, []string{"foo"}},
+		{"search foo bar", []string{}, []string{"foo", "bar"}},
+		{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}},
+		{"search foo\nsearch bar", []string{}, []string{"bar"}},
+		{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}},
+		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}},
+		{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}},
+	}
+	for i, tc := range testCases {
+		ns, srch, err := parseResolvConf(strings.NewReader(tc.data))
+		if err != nil {
+			t.Errorf("expected success, got %v", err)
+			continue
+		}
+		if !reflect.DeepEqual(ns, tc.nameservers) {
+			t.Errorf("[%d] expected nameservers %#v, got %#v", i, tc.nameservers, ns)
+		}
+		if !reflect.DeepEqual(srch, tc.searches) {
+			t.Errorf("[%d] expected searches %#v, got %#v", i, tc.searches, srch)
+		}
+	}
+}

pkg/standalone/standalone.go

@@ -244,6 +244,8 @@ type KubeletConfig struct {
 	RegistryBurst           int
 	MinimumGCAge            time.Duration
 	MaxContainerCount       int
+	ClusterDomain           string
+	ClusterDNS              util.IP
 	EnableServer            bool
 	EnableDebuggingHandlers bool
 	Port                    uint
@@ -265,7 +267,9 @@ func createAndInitKubelet(kc *KubeletConfig, pc *config.PodConfig) *kubelet.Kube
 		kc.RegistryBurst,
 		kc.MinimumGCAge,
 		kc.MaxContainerCount,
-		pc.SeenAllSources)
+		pc.SeenAllSources,
+		kc.ClusterDomain,
+		net.IP(kc.ClusterDNS))
 
 	k.BirthCry()