Add kubelet DNS flags & api disable for DNS

This adds --cluster_dns and --cluster_domain flags to kubelet.  If
non-empty, kubelet will set docker --dns and --dns-search flags based on
these.  It uses the cluster DNS and appends the hosts's DNS servers.
Likewise for DNS search domains.

This also adds API support to bypass cluster DNS entirely, needed to
bootstrap DNS.
This commit is contained in:
Tim Hockin 2014-11-11 21:21:40 -08:00
parent 2a110b097d
commit 652479a3b1
11 changed files with 221 additions and 4 deletions

View File

@ -63,12 +63,15 @@ var (
cAdvisorPort = flag.Uint("cadvisor_port", 4194, "The port of the localhost cAdvisor endpoint")
oomScoreAdj = flag.Int("oom_score_adj", -900, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
apiServerList util.StringList
clusterDomain = flag.String("cluster_domain", "", "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
clusterDNS = util.IP(nil)
)
func init() {
flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
flag.Var(&address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
flag.Var(&clusterDNS, "cluster_dns", "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
}
func setupRunOnce() {
@ -114,6 +117,8 @@ func main() {
RegistryBurst: *registryBurst,
MinimumGCAge: *minimumGCAge,
MaxContainerCount: *maxContainerCount,
ClusterDomain: *clusterDomain,
ClusterDNS: clusterDNS,
Runonce: *runonce,
Port: *port,
CAdvisorPort: *cAdvisorPort,

View File

@ -448,11 +448,27 @@ type PodList struct {
Items []Pod `json:"items"`
}
// DNSPolicy defines how a pod's DNS will be configured.
type DNSPolicy string
const (
// DNSClusterFirst indicates that the pod should use cluster DNS
// first, if it is available, then fall back on the default (as
// determined by kubelet) DNS settings.
DNSClusterFirst DNSPolicy = "ClusterFirst"
// DNSDefault indicates that the pod should use the default (as
// determined by kubelet) DNS settings.
DNSDefault DNSPolicy = "Default"
)
// PodSpec is a description of a pod
type PodSpec struct {
Volumes []Volume `json:"volumes"`
Containers []Container `json:"containers"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
// NodeSelector is a selector which must be true for the pod to fit on a node
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
@ -1029,6 +1045,8 @@ type ContainerManifest struct {
Volumes []Volume `json:"volumes"`
Containers []Container `json:"containers"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy"`
}
// ContainerManifestList is used to communicate container manifests to kubelet.

View File

@ -411,6 +411,7 @@ func init() {
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
return err
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
out.Version = "v1beta2"
return nil
},
@ -424,6 +425,7 @@ func init() {
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
return err
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
return nil
},

View File

@ -60,6 +60,8 @@ type ContainerManifest struct {
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
}
// ContainerManifestList is used to communicate container manifests to kubelet.
@ -827,11 +829,27 @@ type EventList struct {
// Backported from v1beta3 to replace ContainerManifest
// DNSPolicy defines how a pod's DNS will be configured.
type DNSPolicy string
const (
// DNSClusterFirst indicates that the pod should use cluster DNS
// first, if it is available, then fall back on the default (as
// determined by kubelet) DNS settings.
DNSClusterFirst DNSPolicy = "ClusterFirst"
// DNSDefault indicates that the pod should use the default (as
// determined by kubelet) DNS settings.
DNSDefault DNSPolicy = "Default"
)
// PodSpec is a description of a pod
type PodSpec struct {
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// NodeSelector is a selector which must be true for the pod to fit on a node
NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`

View File

@ -286,6 +286,7 @@ func init() {
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
return err
}
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
out.Version = "v1beta2"
return nil
},
@ -299,6 +300,7 @@ func init() {
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
return err
}
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
return nil
},

View File

@ -817,6 +817,8 @@ type ContainerManifest struct {
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
}
// ContainerManifestList is used to communicate container manifests to kubelet.
@ -828,11 +830,27 @@ type ContainerManifestList struct {
// Backported from v1beta3 to replace ContainerManifest
// DNSPolicy defines how a pod's DNS will be configured.
type DNSPolicy string
const (
// DNSClusterFirst indicates that the pod should use cluster DNS
// first, if it is available, then fall back on the default (as
// determined by kubelet) DNS settings.
DNSClusterFirst DNSPolicy = "ClusterFirst"
// DNSDefault indicates that the pod should use the default (as
// determined by kubelet) DNS settings.
DNSDefault DNSPolicy = "Default"
)
// PodSpec is a description of a pod
type PodSpec struct {
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
// NodeSelector is a selector which must be true for the pod to fit on a node
NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`

View File

@ -457,11 +457,27 @@ type RestartPolicy struct {
Never *RestartPolicyNever `json:"never,omitempty"`
}
// DNSPolicy defines how a pod's DNS will be configured.
type DNSPolicy string
const (
// DNSClusterFirst indicates that the pod should use cluster DNS
// first, if it is available, then fall back on the default (as
// determined by kubelet) DNS settings.
DNSClusterFirst DNSPolicy = "ClusterFirst"
// DNSDefault indicates that the pod should use the default (as
// determined by kubelet) DNS settings.
DNSDefault DNSPolicy = "Default"
)
// PodSpec is a description of a pod
type PodSpec struct {
Volumes []Volume `json:"volumes"`
Containers []Container `json:"containers"`
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
// Optional: Set DNS policy. Defaults to "ClusterFirst"
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
// NodeSelector is a selector which must be true for the pod to fit on a node
NodeSelector map[string]string `json:"nodeSelector,omitempty"`

View File

@ -326,6 +326,7 @@ func ValidateManifest(manifest *api.ContainerManifest) errs.ValidationErrorList
allErrs = append(allErrs, vErrs.Prefix("volumes")...)
allErrs = append(allErrs, validateContainers(manifest.Containers, allVolumes).Prefix("containers")...)
allErrs = append(allErrs, validateRestartPolicy(&manifest.RestartPolicy).Prefix("restartPolicy")...)
allErrs = append(allErrs, validateDNSPolicy(&manifest.DNSPolicy).Prefix("dnsPolicy")...)
return allErrs
}
@ -350,6 +351,20 @@ func validateRestartPolicy(restartPolicy *api.RestartPolicy) errs.ValidationErro
return allErrors
}
func validateDNSPolicy(dnsPolicy *api.DNSPolicy) errs.ValidationErrorList {
allErrors := errs.ValidationErrorList{}
switch *dnsPolicy {
case "":
// TODO: move this out to standard defaulting logic, when that is ready.
*dnsPolicy = api.DNSClusterFirst // Default value.
case api.DNSClusterFirst, api.DNSDefault:
break
default:
allErrors = append(allErrors, errs.NewFieldNotSupported("", dnsPolicy))
}
return allErrors
}
// ValidatePod tests if required fields in the pod are set.
func ValidatePod(pod *api.Pod) errs.ValidationErrorList {
allErrs := errs.ValidationErrorList{}
@ -375,6 +390,7 @@ func ValidatePodSpec(spec *api.PodSpec) errs.ValidationErrorList {
allErrs = append(allErrs, vErrs.Prefix("volumes")...)
allErrs = append(allErrs, validateContainers(spec.Containers, allVolumes).Prefix("containers")...)
allErrs = append(allErrs, validateRestartPolicy(&spec.RestartPolicy).Prefix("restartPolicy")...)
allErrs = append(allErrs, validateDNSPolicy(&spec.DNSPolicy).Prefix("dnsPolicy")...)
allErrs = append(allErrs, validateLabels(spec.NodeSelector, "nodeSelector")...)
return allErrs
}

View File

@ -19,6 +19,8 @@ package kubelet
import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"path"
@ -69,7 +71,9 @@ func NewMainKubelet(
pullBurst int,
minimumGCAge time.Duration,
maxContainerCount int,
sourcesReady SourcesReadyFn) *Kubelet {
sourcesReady SourcesReadyFn,
clusterDomain string,
clusterDNS net.IP) *Kubelet {
return &Kubelet{
hostname: hn,
dockerClient: dc,
@ -86,6 +90,8 @@ func NewMainKubelet(
minimumGCAge: minimumGCAge,
maxContainerCount: maxContainerCount,
sourcesReady: sourcesReady,
clusterDomain: clusterDomain,
clusterDNS: clusterDNS,
}
}
@ -138,6 +144,12 @@ type Kubelet struct {
// Optional, minimum age required for garbage collection. If zero, no limit.
minimumGCAge time.Duration
maxContainerCount int
// If non-empty, use this for container DNS search.
clusterDomain string
// If non-nil, use this for container DNS server.
clusterDNS net.IP
}
// GetRootDir returns the full path to the directory under which kubelet can
@ -561,12 +573,18 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
} else if container.Privileged {
return "", fmt.Errorf("container requested privileged mode, but it is disallowed globally.")
}
err = kl.dockerClient.StartContainer(dockerContainer.ID, &docker.HostConfig{
hc := &docker.HostConfig{
PortBindings: portBindings,
Binds: binds,
NetworkMode: netMode,
Privileged: privileged,
})
}
if pod.Spec.DNSPolicy == api.DNSClusterFirst {
if err := kl.applyClusterDNS(hc, pod); err != nil {
return "", err
}
}
err = kl.dockerClient.StartContainer(dockerContainer.ID, hc)
if err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed",
@ -588,6 +606,62 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
return dockertools.DockerID(dockerContainer.ID), err
}
func (kl *Kubelet) applyClusterDNS(hc *docker.HostConfig, pod *api.BoundPod) error {
// Get host DNS settings and append them to cluster DNS settings.
f, err := os.Open("/etc/resolv.conf")
if err != nil {
return err
}
defer f.Close()
hostDNS, hostSearch, err := parseResolvConf(f)
if err != nil {
return err
}
if kl.clusterDNS != nil {
hc.DNS = append([]string{kl.clusterDNS.String()}, hostDNS...)
}
if kl.clusterDomain != "" {
nsDomain := fmt.Sprintf("%s.%s", pod.Namespace, kl.clusterDomain)
hc.DNSSearch = append([]string{nsDomain, kl.clusterDomain}, hostSearch...)
}
return nil
}
// Returns the list of DNS servers and DNS search domains.
func parseResolvConf(reader io.Reader) (nameservers []string, searches []string, err error) {
file, err := ioutil.ReadAll(reader)
if err != nil {
return nil, nil, err
}
// Lines of the form "nameserver 1.2.3.4" accumulate.
nameservers = []string{}
// Lines of the form "search example.com" overrule - last one wins.
searches = []string{}
lines := strings.Split(string(file), "\n")
for l := range lines {
trimmed := strings.TrimSpace(lines[l])
if strings.HasPrefix(trimmed, "#") {
continue
}
fields := strings.Fields(trimmed)
if len(fields) == 0 {
continue
}
if fields[0] == "nameserver" {
nameservers = append(nameservers, fields[1:]...)
}
if fields[0] == "search" {
searches = fields[1:]
}
}
return nameservers, searches, nil
}
// Kill a docker container
func (kl *Kubelet) killContainer(dockerContainer *docker.APIContainers) error {
return kl.killContainerByID(dockerContainer.ID, dockerContainer.Names[0])

View File

@ -1723,3 +1723,47 @@ func TestGarbageCollectImages(t *testing.T) {
t.Errorf("unexpected images removed: %v", fakeDocker.RemovedImages)
}
}
func TestParseResolvConf(t *testing.T) {
testCases := []struct {
data string
nameservers []string
searches []string
}{
{"", []string{}, []string{}},
{" ", []string{}, []string{}},
{"\n", []string{}, []string{}},
{"\t\n\t", []string{}, []string{}},
{"#comment\n", []string{}, []string{}},
{" #comment\n", []string{}, []string{}},
{"#comment\n#comment", []string{}, []string{}},
{"#comment\nnameserver", []string{}, []string{}},
{"#comment\nnameserver\nsearch", []string{}, []string{}},
{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}},
{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}},
{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}},
{"search foo", []string{}, []string{"foo"}},
{"search foo bar", []string{}, []string{"foo", "bar"}},
{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}},
{"search foo\nsearch bar", []string{}, []string{"bar"}},
{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}},
{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}},
{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}},
}
for i, tc := range testCases {
ns, srch, err := parseResolvConf(strings.NewReader(tc.data))
if err != nil {
t.Errorf("expected success, got %v", err)
continue
}
if !reflect.DeepEqual(ns, tc.nameservers) {
t.Errorf("[%d] expected nameservers %#v, got %#v", i, tc.nameservers, ns)
}
if !reflect.DeepEqual(srch, tc.searches) {
t.Errorf("[%d] expected searches %#v, got %#v", i, tc.searches, srch)
}
}
}

View File

@ -244,6 +244,8 @@ type KubeletConfig struct {
RegistryBurst int
MinimumGCAge time.Duration
MaxContainerCount int
ClusterDomain string
ClusterDNS util.IP
EnableServer bool
EnableDebuggingHandlers bool
Port uint
@ -265,7 +267,9 @@ func createAndInitKubelet(kc *KubeletConfig, pc *config.PodConfig) *kubelet.Kube
kc.RegistryBurst,
kc.MinimumGCAge,
kc.MaxContainerCount,
pc.SeenAllSources)
pc.SeenAllSources,
kc.ClusterDomain,
net.IP(kc.ClusterDNS))
k.BirthCry()