From ba6469d47885ae876d4ce67087457315ccc6ebcf Mon Sep 17 00:00:00 2001 From: Abhishek Shah Date: Wed, 21 Oct 2015 10:17:27 -0700 Subject: [PATCH] kubelet manages /etc/hosts file --- pkg/kubelet/dockertools/manager.go | 24 +-- pkg/kubelet/dockertools/manager_test.go | 58 ------ pkg/kubelet/kubelet.go | 58 +++++- pkg/kubelet/kubelet_test.go | 19 +- test/e2e/kubelet_etc_hosts.go | 240 ++++++++++++++++++++++++ 5 files changed, 314 insertions(+), 85 deletions(-) create mode 100644 test/e2e/kubelet_etc_hosts.go diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index e8f1bfc8353..a40c2631c90 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -312,22 +312,6 @@ type containerStatusResult struct { const podIPDownwardAPISelector = "status.podIP" -// podDependsOnIP returns whether any containers in a pod depend on using the pod IP via -// the downward API. -func podDependsOnPodIP(pod *api.Pod) bool { - for _, container := range pod.Spec.Containers { - for _, env := range container.Env { - if env.ValueFrom != nil && - env.ValueFrom.FieldRef != nil && - env.ValueFrom.FieldRef.FieldPath == podIPDownwardAPISelector { - return true - } - } - } - - return false -} - // determineContainerIP determines the IP address of the given container. It is expected // that the container passed is the infrastructure container of a pod and the responsibility // of the caller to ensure that the correct container is passed. @@ -1886,12 +1870,10 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, runningPod kubecontainer.Pod, pod glog.Warningf("Hairpin setup failed for pod %q: %v", podFullName, err) } } - if podDependsOnPodIP(pod) { - // Find the pod IP after starting the infra container in order to expose - // it safely via the downward API without a race. - pod.Status.PodIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer) - } + // Find the pod IP after starting the infra container in order to expose + // it safely via the downward API without a race and be able to use podIP in kubelet-managed /etc/hosts file. + pod.Status.PodIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer) } // Start everything diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index fada0015c48..036291e93d3 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -2059,61 +2059,3 @@ func TestGetIPCMode(t *testing.T) { t.Errorf("expected host ipc mode for pod but got %v", ipcMode) } } - -func TestPodDependsOnPodIP(t *testing.T) { - tests := []struct { - name string - expected bool - env api.EnvVar - }{ - { - name: "depends on pod IP", - expected: true, - env: api.EnvVar{ - Name: "POD_IP", - ValueFrom: &api.EnvVarSource{ - FieldRef: &api.ObjectFieldSelector{ - APIVersion: testapi.Default.Version(), - FieldPath: "status.podIP", - }, - }, - }, - }, - { - name: "literal value", - expected: false, - env: api.EnvVar{ - Name: "SOME_VAR", - Value: "foo", - }, - }, - { - name: "other downward api field", - expected: false, - env: api.EnvVar{ - Name: "POD_NAME", - ValueFrom: &api.EnvVarSource{ - FieldRef: &api.ObjectFieldSelector{ - APIVersion: testapi.Default.Version(), - FieldPath: "metadata.name", - }, - }, - }, - }, - } - - for _, tc := range tests { - pod := &api.Pod{ - Spec: api.PodSpec{ - Containers: []api.Container{ - {Env: []api.EnvVar{tc.env}}, - }, - }, - } - - result := podDependsOnPodIP(pod) - if e, a := tc.expected, result; e != a { - t.Errorf("%v: Unexpected result; expected %v, got %v", tc.name, e, a) - } - } -} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 5a0562842a7..09003f44a8d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -20,6 +20,7 @@ package kubelet // contrib/mesos/pkg/executor/. import ( + "bytes" "errors" "fmt" "io" @@ -102,6 +103,8 @@ const ( // Minimum period for performing global cleanup tasks, i.e., housekeeping // will not be performed more than once per housekeepingMinimumPeriod. housekeepingMinimumPeriod = time.Second * 2 + + etcHostsPath = "/etc/hosts" ) var ( @@ -955,8 +958,17 @@ func (kl *Kubelet) syncNodeStatus() { } } -func makeMounts(container *api.Container, podVolumes kubecontainer.VolumeMap) (mounts []kubecontainer.Mount) { +func makeMounts(pod *api.Pod, podDir string, container *api.Container, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) { + // Kubernetes only mounts on /etc/hosts if : + // - container does not use hostNetwork and + // - container is not a infrastructure(pause) container + // - container is not already mounting on /etc/hosts + // When the pause container is being created, its IP is still unknown. Hence, PodIP will not have been set. + mountEtcHostsFile := !pod.Spec.SecurityContext.HostNetwork && len(pod.Status.PodIP) > 0 + glog.V(4).Infof("Will create hosts mount for container:%q, podIP:%s: %v", container.Name, pod.Status.PodIP, mountEtcHostsFile) + mounts := []kubecontainer.Mount{} for _, mount := range container.VolumeMounts { + mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath) vol, ok := podVolumes[mount.Name] if !ok { glog.Warningf("Mount cannot be satisified for container %q, because the volume is missing: %q", container.Name, mount) @@ -969,7 +981,44 @@ func makeMounts(container *api.Container, podVolumes kubecontainer.VolumeMap) (m ReadOnly: mount.ReadOnly, }) } - return + if mountEtcHostsFile { + hostsMount, err := makeHostsMount(podDir, pod.Status.PodIP, pod.Name) + if err != nil { + return nil, err + } + mounts = append(mounts, *hostsMount) + } + return mounts, nil +} + +func makeHostsMount(podDir, podIP, podName string) (*kubecontainer.Mount, error) { + hostsFilePath := path.Join(podDir, "etc-hosts") + if err := ensureHostsFile(hostsFilePath, podIP, podName); err != nil { + return nil, err + } + return &kubecontainer.Mount{ + Name: "k8s-managed-etc-hosts", + ContainerPath: etcHostsPath, + HostPath: hostsFilePath, + ReadOnly: false, + }, nil +} + +func ensureHostsFile(fileName string, hostIP, hostName string) error { + if _, err := os.Stat(fileName); os.IsExist(err) { + glog.V(4).Infof("kubernetes-managed etc-hosts file exits. Will not be recreated: %q", fileName) + return nil + } + var buffer bytes.Buffer + buffer.WriteString("# Kubernetes-managed hosts file.\n") + buffer.WriteString("127.0.0.1\tlocalhost\n") // ipv4 localhost + buffer.WriteString("::1\tlocalhost ip6-localhost ip6-loopback\n") // ipv6 localhost + buffer.WriteString("fe00::0\tip6-localnet\n") + buffer.WriteString("fe00::0\tip6-mcastprefix\n") + buffer.WriteString("fe00::1\tip6-allnodes\n") + buffer.WriteString("fe00::2\tip6-allrouters\n") + buffer.WriteString(fmt.Sprintf("%s\t%s\n", hostIP, hostName)) + return ioutil.WriteFile(fileName, buffer.Bytes(), 0644) } func makePortMappings(container *api.Container) (ports []kubecontainer.PortMapping) { @@ -1014,7 +1063,10 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont } opts.PortMappings = makePortMappings(container) - opts.Mounts = makeMounts(container, vol) + opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, vol) + if err != nil { + return nil, err + } opts.Envs, err = kl.makeEnvironmentVariables(pod, container) if err != nil { return nil, err diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index d42afdb2a01..c4670fd1b64 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -152,6 +152,11 @@ func newTestPods(count int) []*api.Pod { pods := make([]*api.Pod, count) for i := 0; i < count; i++ { pods[i] = &api.Pod{ + Spec: api.PodSpec{ + SecurityContext: &api.PodSecurityContext{ + HostNetwork: true, + }, + }, ObjectMeta: api.ObjectMeta{ Name: fmt.Sprintf("pod%d", i), }, @@ -506,7 +511,7 @@ func TestMakeVolumeMounts(t *testing.T) { container := api.Container{ VolumeMounts: []api.VolumeMount{ { - MountPath: "/mnt/path", + MountPath: "/etc/hosts", Name: "disk", ReadOnly: false, }, @@ -534,12 +539,20 @@ func TestMakeVolumeMounts(t *testing.T) { "disk5": &stubVolume{"/var/lib/kubelet/podID/volumes/empty/disk5"}, } - mounts := makeMounts(&container, podVolumes) + pod := api.Pod{ + Spec: api.PodSpec{ + SecurityContext: &api.PodSecurityContext{ + HostNetwork: true, + }, + }, + } + + mounts, _ := makeMounts(&pod, "/pod", &container, podVolumes) expectedMounts := []kubecontainer.Mount{ { "disk", - "/mnt/path", + "/etc/hosts", "/mnt/disk", false, }, diff --git a/test/e2e/kubelet_etc_hosts.go b/test/e2e/kubelet_etc_hosts.go new file mode 100644 index 00000000000..5d5adb3bf29 --- /dev/null +++ b/test/e2e/kubelet_etc_hosts.go @@ -0,0 +1,240 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + . "github.com/onsi/ginkgo" + api "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/latest" + "k8s.io/kubernetes/pkg/api/unversioned" + client "k8s.io/kubernetes/pkg/client/unversioned" + "strings" +) + +const ( + kubeletEtcHostsImageName = "gcr.io/google_containers/netexec:1.0" + kubeletEtcHostsPodName = "test-pod" + kubeletEtcHostsHostNetworkPodName = "test-host-network-pod" + etcHostsPartialContent = "# Kubernetes-managed hosts file." +) + +type KubeletManagedHostConfig struct { + hostNetworkPod *api.Pod + pod *api.Pod + f *Framework +} + +var _ = Describe("KubeletManagedEtcHosts", func() { + f := NewFramework("e2e-kubelet-etc-hosts") + config := &KubeletManagedHostConfig{ + f: f, + } + + It("should test kubelet managed /etc/hosts file", func() { + By("Setting up the test") + config.setup() + + By("Running the test") + config.verifyEtcHosts() + }) +}) + +func (config *KubeletManagedHostConfig) verifyEtcHosts() { + By("Verifying /etc/hosts of container is kubelet-managed for pod with hostNetwork=false") + stdout := config.getEtcHostsContent(kubeletEtcHostsPodName, "busybox-1") + assertEtcHostsIsKubeletManaged(stdout) + stdout = config.getEtcHostsContent(kubeletEtcHostsPodName, "busybox-2") + assertEtcHostsIsKubeletManaged(stdout) + + By("Verifying /etc/hosts of container is not kubelet-managed since container specifies /etc/hosts mount") + stdout = config.getEtcHostsContent(kubeletEtcHostsPodName, "busybox-3") + assertEtcHostsIsNotKubeletManaged(stdout) + + By("Verifying /etc/hosts content of container is not kubelet-managed for pod with hostNetwork=true") + stdout = config.getEtcHostsContent(kubeletEtcHostsHostNetworkPodName, "busybox-1") + assertEtcHostsIsNotKubeletManaged(stdout) + stdout = config.getEtcHostsContent(kubeletEtcHostsHostNetworkPodName, "busybox-2") + assertEtcHostsIsNotKubeletManaged(stdout) +} + +func (config *KubeletManagedHostConfig) setup() { + By("Creating hostNetwork=false pod") + config.createPodWithoutHostNetwork() + + By("Creating hostNetwork=true pod") + config.createPodWithHostNetwork() +} + +func (config *KubeletManagedHostConfig) createPodWithoutHostNetwork() { + podSpec := config.createPodSpec(kubeletEtcHostsPodName) + config.pod = config.createPod(podSpec) +} + +func (config *KubeletManagedHostConfig) createPodWithHostNetwork() { + podSpec := config.createPodSpecWithHostNetwork(kubeletEtcHostsHostNetworkPodName) + config.hostNetworkPod = config.createPod(podSpec) +} + +func (config *KubeletManagedHostConfig) createPod(podSpec *api.Pod) *api.Pod { + createdPod, err := config.getPodClient().Create(podSpec) + if err != nil { + Failf("Failed to create %s pod: %v", podSpec.Name, err) + } + expectNoError(config.f.WaitForPodRunning(podSpec.Name)) + createdPod, err = config.getPodClient().Get(podSpec.Name) + if err != nil { + Failf("Failed to retrieve %s pod: %v", podSpec.Name, err) + } + return createdPod +} + +func (config *KubeletManagedHostConfig) getPodClient() client.PodInterface { + return config.f.Client.Pods(config.f.Namespace.Name) +} + +func assertEtcHostsIsKubeletManaged(etcHostsContent string) { + isKubeletManaged := strings.Contains(etcHostsContent, etcHostsPartialContent) + if !isKubeletManaged { + Failf("/etc/hosts file should be kubelet managed, but is not: %q", etcHostsContent) + } +} + +func assertEtcHostsIsNotKubeletManaged(etcHostsContent string) { + isKubeletManaged := strings.Contains(etcHostsContent, etcHostsPartialContent) + if isKubeletManaged { + Failf("/etc/hosts file should not be kubelet managed, but is: %q", etcHostsContent) + } +} + +func (config *KubeletManagedHostConfig) getEtcHostsContent(podName, containerName string) string { + cmd := kubectlCmd("exec", fmt.Sprintf("--namespace=%v", config.f.Namespace.Name), podName, "-c", containerName, "cat", "/etc/hosts") + stdout, stderr, err := startCmdAndStreamOutput(cmd) + if err != nil { + Failf("Failed to retrieve /etc/hosts, err: %q", err) + } + defer stdout.Close() + defer stderr.Close() + + buf := make([]byte, 1000) + var n int + Logf("reading from `kubectl exec` command's stdout") + if n, err = stdout.Read(buf); err != nil { + Failf("Failed to read from kubectl exec stdout: %v", err) + } + return string(buf[:n]) +} + +func (config *KubeletManagedHostConfig) createPodSpec(podName string) *api.Pod { + pod := &api.Pod{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Pod", + APIVersion: latest.GroupOrDie("").Version, + }, + ObjectMeta: api.ObjectMeta{ + Name: podName, + Namespace: config.f.Namespace.Name, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "busybox-1", + Image: kubeletEtcHostsImageName, + ImagePullPolicy: api.PullIfNotPresent, + Command: []string{ + "sleep", + "900", + }, + }, + { + Name: "busybox-2", + Image: kubeletEtcHostsImageName, + ImagePullPolicy: api.PullIfNotPresent, + Command: []string{ + "sleep", + "900", + }, + }, + { + Name: "busybox-3", + Image: kubeletEtcHostsImageName, + ImagePullPolicy: api.PullIfNotPresent, + Command: []string{ + "sleep", + "900", + }, + VolumeMounts: []api.VolumeMount{ + { + Name: "host-etc-hosts", + MountPath: "/etc/hosts", + }, + }, + }, + }, + Volumes: []api.Volume{ + { + Name: "host-etc-hosts", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: "/etc/hosts", + }, + }, + }, + }, + }, + } + return pod +} + +func (config *KubeletManagedHostConfig) createPodSpecWithHostNetwork(podName string) *api.Pod { + pod := &api.Pod{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Pod", + APIVersion: latest.GroupOrDie("").Version, + }, + ObjectMeta: api.ObjectMeta{ + Name: podName, + Namespace: config.f.Namespace.Name, + }, + Spec: api.PodSpec{ + SecurityContext: &api.PodSecurityContext{ + HostNetwork: true, + }, + Containers: []api.Container{ + { + Name: "busybox-1", + Image: kubeletEtcHostsImageName, + ImagePullPolicy: api.PullIfNotPresent, + Command: []string{ + "sleep", + "900", + }, + }, + { + Name: "busybox-2", + Image: kubeletEtcHostsImageName, + ImagePullPolicy: api.PullIfNotPresent, + Command: []string{ + "sleep", + "900", + }, + }, + }, + }, + } + return pod +}