From fc8a37ec86e3a9139882cb4eeecdafc1c0d2d2dd Mon Sep 17 00:00:00 2001 From: vefimova Date: Fri, 5 Aug 2016 03:19:17 -0500 Subject: [PATCH] Added ability for Docker containers to set usage of dns settings along with hostNetwork is true Introduced chages: 1. Re-writing of the resolv.conf file generated by docker. Cluster dns settings aren't passed anymore to docker api in all cases, not only for pods with host network: the resolver conf will be overwritten after infra-container creation to override docker's behaviour. 2. Added new one dnsPolicy - 'ClusterFirstWithHostNet', so now there are: - ClusterFirstWithHostNet - use dns settings in all cases, i.e. with hostNet=true as well - ClusterFirst - use dns settings unless hostNetwork is true - Default Fixes #17406 --- api/openapi-spec/swagger.json | 2 +- api/swagger-spec/apps_v1alpha1.json | 2 +- api/swagger-spec/apps_v1beta1.json | 2 +- api/swagger-spec/batch_v1.json | 2 +- api/swagger-spec/extensions_v1beta1.json | 2 +- api/swagger-spec/v1.json | 2 +- .../apps/v1beta1/definitions.html | 4 +- docs/api-reference/batch/v1/definitions.html | 2 +- .../extensions/v1beta1/definitions.html | 2 +- docs/api-reference/v1/definitions.html | 2 +- federation/apis/openapi-spec/swagger.json | 2 +- .../apis/swagger-spec/extensions_v1beta1.json | 2 +- pkg/api/types.go | 9 ++- pkg/api/v1/generated.proto | 3 +- pkg/api/v1/types.go | 12 +++- pkg/api/v1/types_swagger_doc_generated.go | 2 +- pkg/api/validation/validation.go | 4 +- pkg/generated/openapi/zz_generated.openapi.go | 2 +- pkg/kubelet/container/helpers.go | 4 +- .../container/testing/fake_runtime_helper.go | 8 +-- pkg/kubelet/dockertools/docker_manager.go | 56 ++++++++++++------- pkg/kubelet/kubelet.go | 12 ++-- pkg/kubelet/kubelet_pods.go | 17 +++--- pkg/kubelet/kubelet_pods_test.go | 37 ++++++++++-- .../kuberuntime/kuberuntime_container.go | 2 +- .../kuberuntime/kuberuntime_sandbox.go | 2 +- pkg/kubelet/rkt/rkt.go | 4 +- staging/src/k8s.io/client-go/pkg/api/types.go | 9 ++- .../client-go/pkg/api/v1/generated.proto | 3 +- .../src/k8s.io/client-go/pkg/api/v1/types.go | 12 +++- .../pkg/api/v1/types_swagger_doc_generated.go | 2 +- 31 files changed, 146 insertions(+), 80 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 52f41e70c62..a1d851e20cc 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -39466,7 +39466,7 @@ } }, "dnsPolicy": { - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\".", + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", "type": "string" }, "hostIPC": { diff --git a/api/swagger-spec/apps_v1alpha1.json b/api/swagger-spec/apps_v1alpha1.json index 1bfa55121d9..d2856559f76 100644 --- a/api/swagger-spec/apps_v1alpha1.json +++ b/api/swagger-spec/apps_v1alpha1.json @@ -1303,7 +1303,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/api/swagger-spec/apps_v1beta1.json b/api/swagger-spec/apps_v1beta1.json index e5cc3a71336..c874686761b 100644 --- a/api/swagger-spec/apps_v1beta1.json +++ b/api/swagger-spec/apps_v1beta1.json @@ -2542,7 +2542,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/api/swagger-spec/batch_v1.json b/api/swagger-spec/batch_v1.json index 7bdfb672c61..e7f431b83c0 100644 --- a/api/swagger-spec/batch_v1.json +++ b/api/swagger-spec/batch_v1.json @@ -1325,7 +1325,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/api/swagger-spec/extensions_v1beta1.json b/api/swagger-spec/extensions_v1beta1.json index c1f2ca60773..35602f4132b 100644 --- a/api/swagger-spec/extensions_v1beta1.json +++ b/api/swagger-spec/extensions_v1beta1.json @@ -6789,7 +6789,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index d3eedb0da34..028ac8d5c98 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -18628,7 +18628,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/docs/api-reference/apps/v1beta1/definitions.html b/docs/api-reference/apps/v1beta1/definitions.html index 6653bf3377b..2c44ff43c73 100755 --- a/docs/api-reference/apps/v1beta1/definitions.html +++ b/docs/api-reference/apps/v1beta1/definitions.html @@ -3568,7 +3568,7 @@ The StatefulSet guarantees that a given network identity will always map to the

dnsPolicy

-

Set DNS policy for containers within the pod. One of ClusterFirst or Default. Defaults to "ClusterFirst".

+

Set DNS policy for containers within the pod. One of ClusterFirstWithHostNet, ClusterFirst or Default. Defaults to "ClusterFirst". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to ClusterFirstWithHostNet.

false

string

@@ -6152,7 +6152,7 @@ Examples:
diff --git a/docs/api-reference/batch/v1/definitions.html b/docs/api-reference/batch/v1/definitions.html index 4b28cf2fdc1..de91476fbe9 100755 --- a/docs/api-reference/batch/v1/definitions.html +++ b/docs/api-reference/batch/v1/definitions.html @@ -4821,7 +4821,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i

dnsPolicy

-

Set DNS policy for containers within the pod. One of ClusterFirst or Default. Defaults to "ClusterFirst".

+

Set DNS policy for containers within the pod. One of ClusterFirstWithHostNet, ClusterFirst or Default. Defaults to "ClusterFirst". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to ClusterFirstWithHostNet.

false

string

diff --git a/docs/api-reference/extensions/v1beta1/definitions.html b/docs/api-reference/extensions/v1beta1/definitions.html index aa2547016b4..0e5aaa2ec91 100755 --- a/docs/api-reference/extensions/v1beta1/definitions.html +++ b/docs/api-reference/extensions/v1beta1/definitions.html @@ -4359,7 +4359,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i

dnsPolicy

-

Set DNS policy for containers within the pod. One of ClusterFirst or Default. Defaults to "ClusterFirst".

+

Set DNS policy for containers within the pod. One of ClusterFirstWithHostNet, ClusterFirst or Default. Defaults to "ClusterFirst". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to ClusterFirstWithHostNet.

false

string

diff --git a/docs/api-reference/v1/definitions.html b/docs/api-reference/v1/definitions.html index f2665aaf119..a214c578817 100755 --- a/docs/api-reference/v1/definitions.html +++ b/docs/api-reference/v1/definitions.html @@ -5086,7 +5086,7 @@ The resulting set of endpoints can be viewed as:

dnsPolicy

-

Set DNS policy for containers within the pod. One of ClusterFirst or Default. Defaults to "ClusterFirst".

+

Set DNS policy for containers within the pod. One of ClusterFirstWithHostNet, ClusterFirst or Default. Defaults to "ClusterFirst". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to ClusterFirstWithHostNet.

false

string

diff --git a/federation/apis/openapi-spec/swagger.json b/federation/apis/openapi-spec/swagger.json index b09de36c899..c6ec1c6d550 100644 --- a/federation/apis/openapi-spec/swagger.json +++ b/federation/apis/openapi-spec/swagger.json @@ -12626,7 +12626,7 @@ } }, "dnsPolicy": { - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\".", + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", "type": "string" }, "hostIPC": { diff --git a/federation/apis/swagger-spec/extensions_v1beta1.json b/federation/apis/swagger-spec/extensions_v1beta1.json index 1f02903e95c..9bcc6b7d581 100644 --- a/federation/apis/swagger-spec/extensions_v1beta1.json +++ b/federation/apis/swagger-spec/extensions_v1beta1.json @@ -4595,7 +4595,7 @@ }, "dnsPolicy": { "type": "string", - "description": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\"." + "description": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'." }, "nodeSelector": { "type": "object", diff --git a/pkg/api/types.go b/pkg/api/types.go index 70172c06cb3..1fa4558be44 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1679,9 +1679,14 @@ type PodList struct { type DNSPolicy string const ( + // DNSClusterFirstWithHostNet indicates that the pod should use cluster DNS + // first, if it is available, then fall back on the default + // (as determined by kubelet) DNS settings. + DNSClusterFirstWithHostNet DNSPolicy = "ClusterFirstWithHostNet" + // DNSClusterFirst indicates that the pod should use cluster DNS - // first, if it is available, then fall back on the default (as - // determined by kubelet) DNS settings. + // first unless hostNetwork is true, if it is available, then + // fall back on the default (as determined by kubelet) DNS settings. DNSClusterFirst DNSPolicy = "ClusterFirst" // DNSDefault indicates that the pod should use the default (as diff --git a/pkg/api/v1/generated.proto b/pkg/api/v1/generated.proto index 73eab9a5c3a..a23b7928fab 100644 --- a/pkg/api/v1/generated.proto +++ b/pkg/api/v1/generated.proto @@ -2607,8 +2607,9 @@ message PodSpec { optional int64 activeDeadlineSeconds = 5; // Set DNS policy for containers within the pod. - // One of 'ClusterFirst' or 'Default'. + // One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. // Defaults to "ClusterFirst". + // To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. // +optional optional string dnsPolicy = 6; diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index 1961f367df8..64842ed6cd5 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -1888,9 +1888,14 @@ const ( type DNSPolicy string const ( + // DNSClusterFirstWithHostNet indicates that the pod should use cluster DNS + // first, if it is available, then fall back on the default + // (as determined by kubelet) DNS settings. + DNSClusterFirstWithHostNet DNSPolicy = "ClusterFirstWithHostNet" + // DNSClusterFirst indicates that the pod should use cluster DNS - // first, if it is available, then fall back on the default (as - // determined by kubelet) DNS settings. + // first unless hostNetwork is true, if it is available, then + // fall back on the default (as determined by kubelet) DNS settings. DNSClusterFirst DNSPolicy = "ClusterFirst" // DNSDefault indicates that the pod should use the default (as @@ -2245,8 +2250,9 @@ type PodSpec struct { // +optional ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" protobuf:"varint,5,opt,name=activeDeadlineSeconds"` // Set DNS policy for containers within the pod. - // One of 'ClusterFirst' or 'Default'. + // One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. // Defaults to "ClusterFirst". + // To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. // +optional DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" protobuf:"bytes,6,opt,name=dnsPolicy,casttype=DNSPolicy"` // NodeSelector is a selector which must be true for the pod to fit on a node. diff --git a/pkg/api/v1/types_swagger_doc_generated.go b/pkg/api/v1/types_swagger_doc_generated.go index bc7d261501c..c093bb1d7b0 100644 --- a/pkg/api/v1/types_swagger_doc_generated.go +++ b/pkg/api/v1/types_swagger_doc_generated.go @@ -1321,7 +1321,7 @@ var map_PodSpec = map[string]string{ "restartPolicy": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. Default to Always. More info: http://kubernetes.io/docs/user-guide/pod-states#restartpolicy", "terminationGracePeriodSeconds": "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.", "activeDeadlineSeconds": "Optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers. Value must be a positive integer.", - "dnsPolicy": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\".", + "dnsPolicy": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", "nodeSelector": "NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: http://kubernetes.io/docs/user-guide/node-selection/README", "serviceAccountName": "ServiceAccountName is the name of the ServiceAccount to use to run this pod. More info: http://releases.k8s.io/HEAD/docs/design/service_accounts.md", "serviceAccount": "DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.", diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 0f8599313a7..51c9e97729f 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1788,12 +1788,12 @@ func validateRestartPolicy(restartPolicy *api.RestartPolicy, fldPath *field.Path func validateDNSPolicy(dnsPolicy *api.DNSPolicy, fldPath *field.Path) field.ErrorList { allErrors := field.ErrorList{} switch *dnsPolicy { - case api.DNSClusterFirst, api.DNSDefault: + case api.DNSClusterFirstWithHostNet, api.DNSClusterFirst, api.DNSDefault: break case "": allErrors = append(allErrors, field.Required(fldPath, "")) default: - validValues := []string{string(api.DNSClusterFirst), string(api.DNSDefault)} + validValues := []string{string(api.DNSClusterFirstWithHostNet), string(api.DNSClusterFirst), string(api.DNSDefault)} allErrors = append(allErrors, field.NotSupported(fldPath, dnsPolicy, validValues)) } return allErrors diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a925675cb34..37ff83207ca 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -6491,7 +6491,7 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope }, "dnsPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\".", + Description: "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", Type: []string{"string"}, Format: "", }, diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 12a742a961a..b5c5634d445 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -49,8 +49,8 @@ type HandlerRunner interface { // RuntimeHelper wraps kubelet to make container runtime // able to get necessary informations like the RunContainerOptions, DNS settings. type RuntimeHelper interface { - GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*RunContainerOptions, error) - GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, err error) + GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, useClusterFirstPolicy bool, err error) + GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, useClusterFirstPolicy bool, err error) // GetPodCgroupParent returns the the CgroupName identifer, and its literal cgroupfs form on the host // of a pod. GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) diff --git a/pkg/kubelet/container/testing/fake_runtime_helper.go b/pkg/kubelet/container/testing/fake_runtime_helper.go index 5ef2b782e9c..c81c21de2df 100644 --- a/pkg/kubelet/container/testing/fake_runtime_helper.go +++ b/pkg/kubelet/container/testing/fake_runtime_helper.go @@ -33,20 +33,20 @@ type FakeRuntimeHelper struct { Err error } -func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) { +func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { var opts kubecontainer.RunContainerOptions if len(container.TerminationMessagePath) != 0 { opts.PodContainerDir = f.PodContainerDir } - return &opts, nil + return &opts, false, nil } func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) { return "", "" } -func (f *FakeRuntimeHelper) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) { - return f.DNSServers, f.DNSSearches, f.Err +func (f *FakeRuntimeHelper) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error) { + return f.DNSServers, f.DNSSearches, false, f.Err } // This is not used by docker runtime. diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index cb762893dee..14371bb264a 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -858,6 +858,8 @@ func (dm *DockerManager) runContainer( // setInfraContainerNetworkConfig sets the network configuration for the infra-container. We only set network configuration for infra-container, all // the user containers will share the same network namespace with infra-container. +// NOTE: cluster dns settings aren't passed anymore to docker api in all cases, not only for pods with host network: +// the resolver conf will be overwritten after infra-container creation to override docker's behaviour func setInfraContainerNetworkConfig(pod *v1.Pod, netMode string, opts *kubecontainer.RunContainerOptions, dockerOpts *dockertypes.ContainerCreateConfig) { exposedPorts, portBindings := makePortsAndBindings(opts.PortMappings) dockerOpts.Config.ExposedPorts = exposedPorts @@ -865,12 +867,6 @@ func setInfraContainerNetworkConfig(pod *v1.Pod, netMode string, opts *kubeconta if netMode != namespaceModeHost { dockerOpts.Config.Hostname = opts.Hostname - if len(opts.DNS) > 0 { - dockerOpts.HostConfig.DNS = opts.DNS - } - if len(opts.DNSSearch) > 0 { - dockerOpts.HostConfig.DNSSearch = opts.DNSSearch - } } } @@ -1759,7 +1755,7 @@ func (dm *DockerManager) runContainerInPod(pod *v1.Pod, container *v1.Container, glog.V(5).Infof("Generating ref for container %s: %#v", container.Name, ref) } - opts, err := dm.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) + opts, useClusterFirstPolicy, err := dm.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) if err != nil { return kubecontainer.ContainerID{}, fmt.Errorf("GenerateRunContainerOptions: %v", err) } @@ -1813,14 +1809,15 @@ func (dm *DockerManager) runContainerInPod(pod *v1.Pod, container *v1.Container, return kubecontainer.ContainerID{}, err } - // The addNDotsOption call appends the ndots option to the resolv.conf file generated by docker. + // Re-write resolv.conf file generated by docker. + // NOTE: cluster dns settings aren't passed anymore to docker api in all cases, not only for pods with host network: + // the resolver conf will be overwritten after infra-container creation to override docker's behaviour // This resolv.conf file is shared by all containers of the same pod, and needs to be modified only once per pod. // we modify it when the pause container is created since it is the first container created in the pod since it holds // the networking namespace. - if container.Name == PodInfraContainerName && utsMode != namespaceModeHost { - err = addNDotsOption(containerInfo.ResolvConfPath) - if err != nil { - return kubecontainer.ContainerID{}, fmt.Errorf("addNDotsOption: %v", err) + if container.Name == PodInfraContainerName { + if err := rewriteResolvFile(containerInfo.ResolvConfPath, opts.DNS, opts.DNSSearch, useClusterFirstPolicy); err != nil { + return kubecontainer.ContainerID{}, err } } @@ -1885,7 +1882,7 @@ func (dm *DockerManager) checkDockerAPIVersion(expectedVersion string) (int, err return result, nil } -func addNDotsOption(resolvFilePath string) error { +func rewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, useClusterFirstPolicy bool) error { if len(resolvFilePath) == 0 { glog.Errorf("ResolvConfPath is empty.") return nil @@ -1895,23 +1892,42 @@ func addNDotsOption(resolvFilePath string) error { return fmt.Errorf("ResolvConfPath %q does not exist", resolvFilePath) } - glog.V(4).Infof("DNS ResolvConfPath exists: %s. Will attempt to add ndots option: %s", resolvFilePath, ndotsDNSOption) + var resolvFileContent []string - if err := appendToFile(resolvFilePath, ndotsDNSOption); err != nil { - glog.Errorf("resolv.conf could not be updated: %v", err) - return err + for _, srv := range dns { + resolvFileContent = append(resolvFileContent, "nameserver "+srv) } + + if len(dnsSearch) > 0 { + resolvFileContent = append(resolvFileContent, "search "+strings.Join(dnsSearch, " ")) + } + + if len(resolvFileContent) > 0 { + if useClusterFirstPolicy { + resolvFileContent = append(resolvFileContent, ndotsDNSOption) + } + + resolvFileContentStr := strings.Join(resolvFileContent, "\n") + resolvFileContentStr += "\n" + + glog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent) + if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil { + glog.Errorf("resolv.conf could not be updated: %v", err) + return err + } + } + return nil } -func appendToFile(filePath, stringToAppend string) error { - f, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, 0644) +func rewriteFile(filePath, stringToWrite string) error { + f, err := os.OpenFile(filePath, os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { return err } defer f.Close() - _, err = f.WriteString(stringToAppend) + _, err = f.WriteString(stringToWrite) return err } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 0f155389cbe..7107cc87d09 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1268,22 +1268,22 @@ func (kl *Kubelet) GetKubeClient() clientset.Interface { // GetClusterDNS returns a list of the DNS servers and a list of the DNS search // domains of the cluster. -func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) { +func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error) { var hostDNS, hostSearch []string // Get host DNS settings if kl.resolverConfig != "" { f, err := os.Open(kl.resolverConfig) if err != nil { - return nil, nil, err + return nil, nil, false, err } defer f.Close() hostDNS, hostSearch, err = kl.parseResolvConf(f) if err != nil { - return nil, nil, err + return nil, nil, false, err } } - useClusterFirstPolicy := pod.Spec.DNSPolicy == v1.DNSClusterFirst + useClusterFirstPolicy := ((pod.Spec.DNSPolicy == v1.DNSClusterFirst && !kubecontainer.IsHostNetworkPod(pod)) || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet) if useClusterFirstPolicy && len(kl.clusterDNS) == 0 { // clusterDNS is not known. // pod with ClusterDNSFirst Policy cannot be created @@ -1309,7 +1309,7 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) { } else { hostSearch = kl.formDNSSearchForDNSDefault(hostSearch, pod) } - return hostDNS, hostSearch, nil + return hostDNS, hostSearch, useClusterFirstPolicy, nil } // for a pod with DNSClusterFirst policy, the cluster DNS server is the only nameserver configured for @@ -1321,7 +1321,7 @@ func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, error) { } dnsSearch := kl.formDNSSearch(hostSearch, pod) - return dns, dnsSearch, nil + return dns, dnsSearch, useClusterFirstPolicy, nil } // syncPod is the transaction script for the sync of a single pod. diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index a6e85c723f5..0b5f92ff7e8 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -274,13 +274,14 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) { // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by // the container runtime to set parameters for launching a container. -func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) { +func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { var err error + useClusterFirstPolicy := false _, cgroupParent := kl.GetPodCgroupParent(pod) opts := &kubecontainer.RunContainerOptions{CgroupParent: cgroupParent} hostname, hostDomainName, err := kl.GeneratePodHostNameAndDomain(pod) if err != nil { - return nil, err + return nil, false, err } opts.Hostname = hostname podName := volumehelper.GetUniquePodName(pod) @@ -290,16 +291,16 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai // TODO(random-liu): Move following convert functions into pkg/kubelet/container opts.Devices, err = kl.makeDevices(pod, container) if err != nil { - return nil, err + return nil, false, err } opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) if err != nil { - return nil, err + return nil, false, err } opts.Envs, err = kl.makeEnvironmentVariables(pod, container, podIP) if err != nil { - return nil, err + return nil, false, err } // Disabling adding TerminationMessagePath on Windows as these files would be mounted as docker volume and @@ -313,9 +314,9 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai } } - opts.DNS, opts.DNSSearch, err = kl.GetClusterDNS(pod) + opts.DNS, opts.DNSSearch, useClusterFirstPolicy, err = kl.GetClusterDNS(pod) if err != nil { - return nil, err + return nil, false, err } // only do this check if the experimental behavior is enabled, otherwise allow it to default to false @@ -323,7 +324,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai opts.EnableHostUserNamespace = kl.enableHostUserNamespace(pod) } - return opts, nil + return opts, useClusterFirstPolicy, nil } var masterServices = sets.NewString("kubernetes") diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 7c7a2cb7068..78fa37bb5b4 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -176,14 +176,17 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) { kubelet.clusterDomain = "kubernetes.io" kubelet.clusterDNS = []net.IP{net.ParseIP(clusterNS)} - pods := newTestPods(2) - pods[0].Spec.DNSPolicy = v1.DNSClusterFirst - pods[1].Spec.DNSPolicy = v1.DNSDefault + pods := newTestPods(4) + pods[0].Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + pods[1].Spec.DNSPolicy = v1.DNSClusterFirst + pods[2].Spec.DNSPolicy = v1.DNSClusterFirst + pods[2].Spec.HostNetwork = false + pods[3].Spec.DNSPolicy = v1.DNSDefault - options := make([]*kubecontainer.RunContainerOptions, 2) + options := make([]*kubecontainer.RunContainerOptions, 4) for i, pod := range pods { var err error - options[i], err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") + options[i], _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") if err != nil { t.Fatalf("failed to generate container options: %v", err) } @@ -200,11 +203,23 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) { if len(options[1].DNSSearch) != 1 || options[1].DNSSearch[0] != "." { t.Errorf("expected search \".\", got %+v", options[1].DNSSearch) } + if len(options[2].DNS) != 1 || options[2].DNS[0] != clusterNS { + t.Errorf("expected nameserver %s, got %+v", clusterNS, options[2].DNS) + } + if len(options[2].DNSSearch) == 0 || options[2].DNSSearch[0] != ".svc."+kubelet.clusterDomain { + t.Errorf("expected search %s, got %+v", ".svc."+kubelet.clusterDomain, options[2].DNSSearch) + } + if len(options[3].DNS) != 1 || options[3].DNS[0] != "127.0.0.1" { + t.Errorf("expected nameserver 127.0.0.1, got %+v", options[3].DNS) + } + if len(options[3].DNSSearch) != 1 || options[3].DNSSearch[0] != "." { + t.Errorf("expected search \".\", got %+v", options[3].DNSSearch) + } kubelet.resolverConfig = "/etc/resolv.conf" for i, pod := range pods { var err error - options[i], err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") + options[i], _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") if err != nil { t.Fatalf("failed to generate container options: %v", err) } @@ -224,6 +239,16 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) { } else if options[0].DNSSearch[0] != ".svc."+kubelet.clusterDomain { t.Errorf("expected domain %s, got %s", ".svc."+kubelet.clusterDomain, options[0].DNSSearch) } + if len(options[2].DNS) != 1 { + t.Errorf("expected cluster nameserver only, got %+v", options[2].DNS) + } else if options[2].DNS[0] != clusterNS { + t.Errorf("expected nameserver %s, got %v", clusterNS, options[2].DNS[0]) + } + if len(options[2].DNSSearch) != expLength { + t.Errorf("expected prepend of cluster domain, got %+v", options[2].DNSSearch) + } else if options[2].DNSSearch[0] != ".svc."+kubelet.clusterDomain { + t.Errorf("expected domain %s, got %s", ".svc."+kubelet.clusterDomain, options[0].DNSSearch) + } } type testServiceLister struct { diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 474407a9165..5c519354e58 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -132,7 +132,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb // generateContainerConfig generates container config for kubelet runtime v1. func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string) (*runtimeapi.ContainerConfig, error) { - opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) + opts, _, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) if err != nil { return nil, err } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go index 3980604aab6..88e20f98c0d 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go @@ -75,7 +75,7 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp } if !kubecontainer.IsHostNetworkPod(pod) { - dnsServers, dnsSearches, err := m.runtimeHelper.GetClusterDNS(pod) + dnsServers, dnsSearches, _, err := m.runtimeHelper.GetClusterDNS(pod) if err != nil { return nil, err } diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 4a125306e38..2d9817f173e 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -810,7 +810,7 @@ func (r *Runtime) newAppcRuntimeApp(pod *v1.Pod, podIP string, c v1.Container, r } // TODO: determine how this should be handled for rkt - opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) + opts, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) if err != nil { return err } @@ -1009,7 +1009,7 @@ func (r *Runtime) generateRunCommand(pod *v1.Pod, uuid, netnsName string) (strin } } else { // Setup DNS. - dnsServers, dnsSearches, err := r.runtimeHelper.GetClusterDNS(pod) + dnsServers, dnsSearches, _, err := r.runtimeHelper.GetClusterDNS(pod) if err != nil { return "", err } diff --git a/staging/src/k8s.io/client-go/pkg/api/types.go b/staging/src/k8s.io/client-go/pkg/api/types.go index 70172c06cb3..1fa4558be44 100644 --- a/staging/src/k8s.io/client-go/pkg/api/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/types.go @@ -1679,9 +1679,14 @@ type PodList struct { type DNSPolicy string const ( + // DNSClusterFirstWithHostNet indicates that the pod should use cluster DNS + // first, if it is available, then fall back on the default + // (as determined by kubelet) DNS settings. + DNSClusterFirstWithHostNet DNSPolicy = "ClusterFirstWithHostNet" + // DNSClusterFirst indicates that the pod should use cluster DNS - // first, if it is available, then fall back on the default (as - // determined by kubelet) DNS settings. + // first unless hostNetwork is true, if it is available, then + // fall back on the default (as determined by kubelet) DNS settings. DNSClusterFirst DNSPolicy = "ClusterFirst" // DNSDefault indicates that the pod should use the default (as diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/generated.proto b/staging/src/k8s.io/client-go/pkg/api/v1/generated.proto index 73eab9a5c3a..a23b7928fab 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/generated.proto +++ b/staging/src/k8s.io/client-go/pkg/api/v1/generated.proto @@ -2607,8 +2607,9 @@ message PodSpec { optional int64 activeDeadlineSeconds = 5; // Set DNS policy for containers within the pod. - // One of 'ClusterFirst' or 'Default'. + // One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. // Defaults to "ClusterFirst". + // To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. // +optional optional string dnsPolicy = 6; diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/types.go b/staging/src/k8s.io/client-go/pkg/api/v1/types.go index 1961f367df8..64842ed6cd5 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/types.go @@ -1888,9 +1888,14 @@ const ( type DNSPolicy string const ( + // DNSClusterFirstWithHostNet indicates that the pod should use cluster DNS + // first, if it is available, then fall back on the default + // (as determined by kubelet) DNS settings. + DNSClusterFirstWithHostNet DNSPolicy = "ClusterFirstWithHostNet" + // DNSClusterFirst indicates that the pod should use cluster DNS - // first, if it is available, then fall back on the default (as - // determined by kubelet) DNS settings. + // first unless hostNetwork is true, if it is available, then + // fall back on the default (as determined by kubelet) DNS settings. DNSClusterFirst DNSPolicy = "ClusterFirst" // DNSDefault indicates that the pod should use the default (as @@ -2245,8 +2250,9 @@ type PodSpec struct { // +optional ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" protobuf:"varint,5,opt,name=activeDeadlineSeconds"` // Set DNS policy for containers within the pod. - // One of 'ClusterFirst' or 'Default'. + // One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. // Defaults to "ClusterFirst". + // To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. // +optional DNSPolicy DNSPolicy `json:"dnsPolicy,omitempty" protobuf:"bytes,6,opt,name=dnsPolicy,casttype=DNSPolicy"` // NodeSelector is a selector which must be true for the pod to fit on a node. diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/client-go/pkg/api/v1/types_swagger_doc_generated.go index bc7d261501c..c093bb1d7b0 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/types_swagger_doc_generated.go @@ -1321,7 +1321,7 @@ var map_PodSpec = map[string]string{ "restartPolicy": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. Default to Always. More info: http://kubernetes.io/docs/user-guide/pod-states#restartpolicy", "terminationGracePeriodSeconds": "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.", "activeDeadlineSeconds": "Optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers. Value must be a positive integer.", - "dnsPolicy": "Set DNS policy for containers within the pod. One of 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\".", + "dnsPolicy": "Set DNS policy for containers within the pod. One of 'ClusterFirstWithHostNet', 'ClusterFirst' or 'Default'. Defaults to \"ClusterFirst\". To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", "nodeSelector": "NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: http://kubernetes.io/docs/user-guide/node-selection/README", "serviceAccountName": "ServiceAccountName is the name of the ServiceAccount to use to run this pod. More info: http://releases.k8s.io/HEAD/docs/design/service_accounts.md", "serviceAccount": "DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.",