mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 19:31:44 +00:00
Add kubelet DNS flags & api disable for DNS
This adds --cluster_dns and --cluster_domain flags to kubelet. If non-empty, kubelet will set docker --dns and --dns-search flags based on these. It uses the cluster DNS and appends the hosts's DNS servers. Likewise for DNS search domains. This also adds API support to bypass cluster DNS entirely, needed to bootstrap DNS.
This commit is contained in:
parent
2a110b097d
commit
652479a3b1
@ -63,12 +63,15 @@ var (
|
||||
cAdvisorPort = flag.Uint("cadvisor_port", 4194, "The port of the localhost cAdvisor endpoint")
|
||||
oomScoreAdj = flag.Int("oom_score_adj", -900, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
|
||||
apiServerList util.StringList
|
||||
clusterDomain = flag.String("cluster_domain", "", "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
|
||||
clusterDNS = util.IP(nil)
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
|
||||
flag.Var(&address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
|
||||
flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
|
||||
flag.Var(&clusterDNS, "cluster_dns", "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
|
||||
}
|
||||
|
||||
func setupRunOnce() {
|
||||
@ -114,6 +117,8 @@ func main() {
|
||||
RegistryBurst: *registryBurst,
|
||||
MinimumGCAge: *minimumGCAge,
|
||||
MaxContainerCount: *maxContainerCount,
|
||||
ClusterDomain: *clusterDomain,
|
||||
ClusterDNS: clusterDNS,
|
||||
Runonce: *runonce,
|
||||
Port: *port,
|
||||
CAdvisorPort: *cAdvisorPort,
|
||||
|
@ -448,11 +448,27 @@ type PodList struct {
|
||||
Items []Pod `json:"items"`
|
||||
}
|
||||
|
||||
// DNSPolicy defines how a pod's DNS will be configured.
|
||||
type DNSPolicy string
|
||||
|
||||
const (
|
||||
// DNSClusterFirst indicates that the pod should use cluster DNS
|
||||
// first, if it is available, then fall back on the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSClusterFirst DNSPolicy = "ClusterFirst"
|
||||
|
||||
// DNSDefault indicates that the pod should use the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSDefault DNSPolicy = "Default"
|
||||
)
|
||||
|
||||
// PodSpec is a description of a pod
|
||||
type PodSpec struct {
|
||||
Volumes []Volume `json:"volumes"`
|
||||
Containers []Container `json:"containers"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
|
||||
// NodeSelector is a selector which must be true for the pod to fit on a node
|
||||
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
|
||||
|
||||
@ -1029,6 +1045,8 @@ type ContainerManifest struct {
|
||||
Volumes []Volume `json:"volumes"`
|
||||
Containers []Container `json:"containers"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy"`
|
||||
}
|
||||
|
||||
// ContainerManifestList is used to communicate container manifests to kubelet.
|
||||
|
@ -411,6 +411,7 @@ func init() {
|
||||
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
|
||||
out.Version = "v1beta2"
|
||||
return nil
|
||||
},
|
||||
@ -424,6 +425,7 @@ func init() {
|
||||
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
|
||||
return nil
|
||||
},
|
||||
|
||||
|
@ -60,6 +60,8 @@ type ContainerManifest struct {
|
||||
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
|
||||
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
|
||||
}
|
||||
|
||||
// ContainerManifestList is used to communicate container manifests to kubelet.
|
||||
@ -827,11 +829,27 @@ type EventList struct {
|
||||
|
||||
// Backported from v1beta3 to replace ContainerManifest
|
||||
|
||||
// DNSPolicy defines how a pod's DNS will be configured.
|
||||
type DNSPolicy string
|
||||
|
||||
const (
|
||||
// DNSClusterFirst indicates that the pod should use cluster DNS
|
||||
// first, if it is available, then fall back on the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSClusterFirst DNSPolicy = "ClusterFirst"
|
||||
|
||||
// DNSDefault indicates that the pod should use the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSDefault DNSPolicy = "Default"
|
||||
)
|
||||
|
||||
// PodSpec is a description of a pod
|
||||
type PodSpec struct {
|
||||
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
|
||||
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
|
||||
// NodeSelector is a selector which must be true for the pod to fit on a node
|
||||
NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`
|
||||
|
||||
|
@ -286,6 +286,7 @@ func init() {
|
||||
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
out.DNSPolicy = DNSPolicy(in.DNSPolicy)
|
||||
out.Version = "v1beta2"
|
||||
return nil
|
||||
},
|
||||
@ -299,6 +300,7 @@ func init() {
|
||||
if err := s.Convert(&in.RestartPolicy, &out.RestartPolicy, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
out.DNSPolicy = newer.DNSPolicy(in.DNSPolicy)
|
||||
return nil
|
||||
},
|
||||
|
||||
|
@ -817,6 +817,8 @@ type ContainerManifest struct {
|
||||
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
|
||||
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
|
||||
}
|
||||
|
||||
// ContainerManifestList is used to communicate container manifests to kubelet.
|
||||
@ -828,11 +830,27 @@ type ContainerManifestList struct {
|
||||
|
||||
// Backported from v1beta3 to replace ContainerManifest
|
||||
|
||||
// DNSPolicy defines how a pod's DNS will be configured.
|
||||
type DNSPolicy string
|
||||
|
||||
const (
|
||||
// DNSClusterFirst indicates that the pod should use cluster DNS
|
||||
// first, if it is available, then fall back on the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSClusterFirst DNSPolicy = "ClusterFirst"
|
||||
|
||||
// DNSDefault indicates that the pod should use the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSDefault DNSPolicy = "Default"
|
||||
)
|
||||
|
||||
// PodSpec is a description of a pod
|
||||
type PodSpec struct {
|
||||
Volumes []Volume `json:"volumes" description:"list of volumes that can be mounted by containers belonging to the pod"`
|
||||
Containers []Container `json:"containers" description:"list of containers belonging to the pod"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty" description:"restart policy for all containers within the pod; one of RestartPolicyAlways, RestartPolicyOnFailure, RestartPolicyNever"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" description:"DNS policy for containers within the pod; one of 'ClusterFirst' or 'Default'"`
|
||||
// NodeSelector is a selector which must be true for the pod to fit on a node
|
||||
NodeSelector map[string]string `json:"nodeSelector,omitempty" description:"selector which must match a node's labels for the pod to be scheduled on that node"`
|
||||
|
||||
|
@ -457,11 +457,27 @@ type RestartPolicy struct {
|
||||
Never *RestartPolicyNever `json:"never,omitempty"`
|
||||
}
|
||||
|
||||
// DNSPolicy defines how a pod's DNS will be configured.
|
||||
type DNSPolicy string
|
||||
|
||||
const (
|
||||
// DNSClusterFirst indicates that the pod should use cluster DNS
|
||||
// first, if it is available, then fall back on the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSClusterFirst DNSPolicy = "ClusterFirst"
|
||||
|
||||
// DNSDefault indicates that the pod should use the default (as
|
||||
// determined by kubelet) DNS settings.
|
||||
DNSDefault DNSPolicy = "Default"
|
||||
)
|
||||
|
||||
// PodSpec is a description of a pod
|
||||
type PodSpec struct {
|
||||
Volumes []Volume `json:"volumes"`
|
||||
Containers []Container `json:"containers"`
|
||||
RestartPolicy RestartPolicy `json:"restartPolicy,omitempty"`
|
||||
// Optional: Set DNS policy. Defaults to "ClusterFirst"
|
||||
DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty"`
|
||||
// NodeSelector is a selector which must be true for the pod to fit on a node
|
||||
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
|
||||
|
||||
|
@ -326,6 +326,7 @@ func ValidateManifest(manifest *api.ContainerManifest) errs.ValidationErrorList
|
||||
allErrs = append(allErrs, vErrs.Prefix("volumes")...)
|
||||
allErrs = append(allErrs, validateContainers(manifest.Containers, allVolumes).Prefix("containers")...)
|
||||
allErrs = append(allErrs, validateRestartPolicy(&manifest.RestartPolicy).Prefix("restartPolicy")...)
|
||||
allErrs = append(allErrs, validateDNSPolicy(&manifest.DNSPolicy).Prefix("dnsPolicy")...)
|
||||
return allErrs
|
||||
}
|
||||
|
||||
@ -350,6 +351,20 @@ func validateRestartPolicy(restartPolicy *api.RestartPolicy) errs.ValidationErro
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func validateDNSPolicy(dnsPolicy *api.DNSPolicy) errs.ValidationErrorList {
|
||||
allErrors := errs.ValidationErrorList{}
|
||||
switch *dnsPolicy {
|
||||
case "":
|
||||
// TODO: move this out to standard defaulting logic, when that is ready.
|
||||
*dnsPolicy = api.DNSClusterFirst // Default value.
|
||||
case api.DNSClusterFirst, api.DNSDefault:
|
||||
break
|
||||
default:
|
||||
allErrors = append(allErrors, errs.NewFieldNotSupported("", dnsPolicy))
|
||||
}
|
||||
return allErrors
|
||||
}
|
||||
|
||||
// ValidatePod tests if required fields in the pod are set.
|
||||
func ValidatePod(pod *api.Pod) errs.ValidationErrorList {
|
||||
allErrs := errs.ValidationErrorList{}
|
||||
@ -375,6 +390,7 @@ func ValidatePodSpec(spec *api.PodSpec) errs.ValidationErrorList {
|
||||
allErrs = append(allErrs, vErrs.Prefix("volumes")...)
|
||||
allErrs = append(allErrs, validateContainers(spec.Containers, allVolumes).Prefix("containers")...)
|
||||
allErrs = append(allErrs, validateRestartPolicy(&spec.RestartPolicy).Prefix("restartPolicy")...)
|
||||
allErrs = append(allErrs, validateDNSPolicy(&spec.DNSPolicy).Prefix("dnsPolicy")...)
|
||||
allErrs = append(allErrs, validateLabels(spec.NodeSelector, "nodeSelector")...)
|
||||
return allErrs
|
||||
}
|
||||
|
@ -19,6 +19,8 @@ package kubelet
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@ -69,7 +71,9 @@ func NewMainKubelet(
|
||||
pullBurst int,
|
||||
minimumGCAge time.Duration,
|
||||
maxContainerCount int,
|
||||
sourcesReady SourcesReadyFn) *Kubelet {
|
||||
sourcesReady SourcesReadyFn,
|
||||
clusterDomain string,
|
||||
clusterDNS net.IP) *Kubelet {
|
||||
return &Kubelet{
|
||||
hostname: hn,
|
||||
dockerClient: dc,
|
||||
@ -86,6 +90,8 @@ func NewMainKubelet(
|
||||
minimumGCAge: minimumGCAge,
|
||||
maxContainerCount: maxContainerCount,
|
||||
sourcesReady: sourcesReady,
|
||||
clusterDomain: clusterDomain,
|
||||
clusterDNS: clusterDNS,
|
||||
}
|
||||
}
|
||||
|
||||
@ -138,6 +144,12 @@ type Kubelet struct {
|
||||
// Optional, minimum age required for garbage collection. If zero, no limit.
|
||||
minimumGCAge time.Duration
|
||||
maxContainerCount int
|
||||
|
||||
// If non-empty, use this for container DNS search.
|
||||
clusterDomain string
|
||||
|
||||
// If non-nil, use this for container DNS server.
|
||||
clusterDNS net.IP
|
||||
}
|
||||
|
||||
// GetRootDir returns the full path to the directory under which kubelet can
|
||||
@ -561,12 +573,18 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
|
||||
} else if container.Privileged {
|
||||
return "", fmt.Errorf("container requested privileged mode, but it is disallowed globally.")
|
||||
}
|
||||
err = kl.dockerClient.StartContainer(dockerContainer.ID, &docker.HostConfig{
|
||||
hc := &docker.HostConfig{
|
||||
PortBindings: portBindings,
|
||||
Binds: binds,
|
||||
NetworkMode: netMode,
|
||||
Privileged: privileged,
|
||||
})
|
||||
}
|
||||
if pod.Spec.DNSPolicy == api.DNSClusterFirst {
|
||||
if err := kl.applyClusterDNS(hc, pod); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
err = kl.dockerClient.StartContainer(dockerContainer.ID, hc)
|
||||
if err != nil {
|
||||
if ref != nil {
|
||||
record.Eventf(ref, "failed", "failed",
|
||||
@ -588,6 +606,62 @@ func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, pod
|
||||
return dockertools.DockerID(dockerContainer.ID), err
|
||||
}
|
||||
|
||||
func (kl *Kubelet) applyClusterDNS(hc *docker.HostConfig, pod *api.BoundPod) error {
|
||||
// Get host DNS settings and append them to cluster DNS settings.
|
||||
f, err := os.Open("/etc/resolv.conf")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hostDNS, hostSearch, err := parseResolvConf(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if kl.clusterDNS != nil {
|
||||
hc.DNS = append([]string{kl.clusterDNS.String()}, hostDNS...)
|
||||
}
|
||||
if kl.clusterDomain != "" {
|
||||
nsDomain := fmt.Sprintf("%s.%s", pod.Namespace, kl.clusterDomain)
|
||||
hc.DNSSearch = append([]string{nsDomain, kl.clusterDomain}, hostSearch...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns the list of DNS servers and DNS search domains.
|
||||
func parseResolvConf(reader io.Reader) (nameservers []string, searches []string, err error) {
|
||||
file, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Lines of the form "nameserver 1.2.3.4" accumulate.
|
||||
nameservers = []string{}
|
||||
|
||||
// Lines of the form "search example.com" overrule - last one wins.
|
||||
searches = []string{}
|
||||
|
||||
lines := strings.Split(string(file), "\n")
|
||||
for l := range lines {
|
||||
trimmed := strings.TrimSpace(lines[l])
|
||||
if strings.HasPrefix(trimmed, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(trimmed)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
if fields[0] == "nameserver" {
|
||||
nameservers = append(nameservers, fields[1:]...)
|
||||
}
|
||||
if fields[0] == "search" {
|
||||
searches = fields[1:]
|
||||
}
|
||||
}
|
||||
return nameservers, searches, nil
|
||||
}
|
||||
|
||||
// Kill a docker container
|
||||
func (kl *Kubelet) killContainer(dockerContainer *docker.APIContainers) error {
|
||||
return kl.killContainerByID(dockerContainer.ID, dockerContainer.Names[0])
|
||||
|
@ -1723,3 +1723,47 @@ func TestGarbageCollectImages(t *testing.T) {
|
||||
t.Errorf("unexpected images removed: %v", fakeDocker.RemovedImages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResolvConf(t *testing.T) {
|
||||
testCases := []struct {
|
||||
data string
|
||||
nameservers []string
|
||||
searches []string
|
||||
}{
|
||||
{"", []string{}, []string{}},
|
||||
{" ", []string{}, []string{}},
|
||||
{"\n", []string{}, []string{}},
|
||||
{"\t\n\t", []string{}, []string{}},
|
||||
{"#comment\n", []string{}, []string{}},
|
||||
{" #comment\n", []string{}, []string{}},
|
||||
{"#comment\n#comment", []string{}, []string{}},
|
||||
{"#comment\nnameserver", []string{}, []string{}},
|
||||
{"#comment\nnameserver\nsearch", []string{}, []string{}},
|
||||
{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
|
||||
{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
|
||||
{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}},
|
||||
{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}},
|
||||
{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}},
|
||||
{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}},
|
||||
{"search foo", []string{}, []string{"foo"}},
|
||||
{"search foo bar", []string{}, []string{"foo", "bar"}},
|
||||
{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}},
|
||||
{"search foo\nsearch bar", []string{}, []string{"bar"}},
|
||||
{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}},
|
||||
{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}},
|
||||
{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
ns, srch, err := parseResolvConf(strings.NewReader(tc.data))
|
||||
if err != nil {
|
||||
t.Errorf("expected success, got %v", err)
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(ns, tc.nameservers) {
|
||||
t.Errorf("[%d] expected nameservers %#v, got %#v", i, tc.nameservers, ns)
|
||||
}
|
||||
if !reflect.DeepEqual(srch, tc.searches) {
|
||||
t.Errorf("[%d] expected searches %#v, got %#v", i, tc.searches, srch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -244,6 +244,8 @@ type KubeletConfig struct {
|
||||
RegistryBurst int
|
||||
MinimumGCAge time.Duration
|
||||
MaxContainerCount int
|
||||
ClusterDomain string
|
||||
ClusterDNS util.IP
|
||||
EnableServer bool
|
||||
EnableDebuggingHandlers bool
|
||||
Port uint
|
||||
@ -265,7 +267,9 @@ func createAndInitKubelet(kc *KubeletConfig, pc *config.PodConfig) *kubelet.Kube
|
||||
kc.RegistryBurst,
|
||||
kc.MinimumGCAge,
|
||||
kc.MaxContainerCount,
|
||||
pc.SeenAllSources)
|
||||
pc.SeenAllSources,
|
||||
kc.ClusterDomain,
|
||||
net.IP(kc.ClusterDNS))
|
||||
|
||||
k.BirthCry()
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user