From dfb6dd010f5efe0ba0a5c7225fcba53aafb52c93 Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Thu, 14 Apr 2016 12:00:51 -0700 Subject: [PATCH] rkt: Fix hostnetwork. Mount hosts' /etc/hosts, /etc/resolv.conf, set host's hostname when running the pod in the host's network. Besides, do not set the DNS flags when running in host's network. --- pkg/kubelet/container/helpers.go | 5 ++ pkg/kubelet/dockertools/manager.go | 13 ++--- pkg/kubelet/rkt/rkt.go | 86 ++++++++++++++++++++++-------- pkg/kubelet/rkt/rkt_test.go | 13 +++-- 4 files changed, 83 insertions(+), 34 deletions(-) diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index b62232fc129..d70f3516872 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -179,3 +179,8 @@ func (irecorder *innerEventRecorder) PastEventf(object runtime.Object, timestamp irecorder.recorder.PastEventf(ref, timestamp, eventtype, reason, messageFmt, args...) } } + +// Pod must not be nil. +func IsHostNetworkPod(pod *api.Pod) bool { + return pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.HostNetwork +} diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 6b9699ba708..b40e6af0158 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -825,7 +825,7 @@ func (dm *DockerManager) podInfraContainerChanged(pod *api.Pod, podInfraContaine var ports []api.ContainerPort // Check network mode. - if usesHostNetwork(pod) { + if kubecontainer.IsHostNetworkPod(pod) { dockerPodInfraContainer, err := dm.client.InspectContainer(podInfraContainerStatus.ID.ID) if err != nil { return false, err @@ -853,11 +853,6 @@ func (dm *DockerManager) podInfraContainerChanged(pod *api.Pod, podInfraContaine return podInfraContainerStatus.Hash != kubecontainer.HashContainer(expectedPodInfraContainer), nil } -// pod must not be nil -func usesHostNetwork(pod *api.Pod) bool { - return pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.HostNetwork -} - // determine if the container root should be a read only filesystem. func readOnlyRootFilesystem(container *api.Container) bool { return container.SecurityContext != nil && container.SecurityContext.ReadOnlyRootFilesystem != nil && *container.SecurityContext.ReadOnlyRootFilesystem @@ -1484,7 +1479,7 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe } utsMode := "" - if usesHostNetwork(pod) { + if kubecontainer.IsHostNetworkPod(pod) { utsMode = namespaceModeHost } @@ -1653,7 +1648,7 @@ func (dm *DockerManager) createPodInfraContainer(pod *api.Pod) (kubecontainer.Do netNamespace := "" var ports []api.ContainerPort - if usesHostNetwork(pod) { + if kubecontainer.IsHostNetworkPod(pod) { netNamespace = namespaceModeHost } else if dm.networkPlugin.Name() == "cni" || dm.networkPlugin.Name() == "kubenet" { netNamespace = "none" @@ -1906,7 +1901,7 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec setupNetworkResult := kubecontainer.NewSyncResult(kubecontainer.SetupNetwork, kubecontainer.GetPodFullName(pod)) result.AddSyncResult(setupNetworkResult) - if !usesHostNetwork(pod) { + if !kubecontainer.IsHostNetworkPod(pod) { // Call the networking plugin err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) if err != nil { diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 4934acab146..5be1f3cf0eb 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -569,6 +569,24 @@ func (r *Runtime) makePodManifest(pod *api.Pod, pullSecrets []api.Secret) (*appc return manifest, nil } +// TODO(yifan): Can make rkt handle this when '--net=host'. See https://github.com/coreos/rkt/issues/2430. +func makeHostNetworkMount(opts *kubecontainer.RunContainerOptions) (*kubecontainer.Mount, *kubecontainer.Mount) { + hostsMount := kubecontainer.Mount{ + Name: "kubernetes-hostnetwork-hosts-conf", + ContainerPath: "/etc/hosts", + HostPath: "/etc/hosts", + ReadOnly: true, + } + resolvMount := kubecontainer.Mount{ + Name: "kubernetes-hostnetwork-resolv-conf", + ContainerPath: "/etc/resolv.conf", + HostPath: "/etc/resolv.conf", + ReadOnly: true, + } + opts.Mounts = append(opts.Mounts, hostsMount, resolvMount) + return &hostsMount, &resolvMount +} + func makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *api.Container) (*kubecontainer.Mount, error) { if opts.PodContainerDir == "" || container.TerminationMessagePath == "" { return nil, nil @@ -590,7 +608,7 @@ func makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *a return nil, err } - mnt := &kubecontainer.Mount{ + mnt := kubecontainer.Mount{ // Use a random name for the termination message mount, so that // when a container restarts, it will not overwrite the old termination // message. @@ -599,9 +617,9 @@ func makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *a HostPath: containerLogPath, ReadOnly: false, } - opts.Mounts = append(opts.Mounts, *mnt) + opts.Mounts = append(opts.Mounts, mnt) - return mnt, nil + return &mnt, nil } func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, c api.Container, pullSecrets []api.Secret, manifest *appcschema.PodManifest) error { @@ -638,6 +656,23 @@ func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, c api.Container, pullSecrets [ return err } + // If run in 'hostnetwork' mode, then mount the host's /etc/resolv.conf and /etc/hosts, + // and add volumes. + var hostsMnt, resolvMnt *kubecontainer.Mount + if kubecontainer.IsHostNetworkPod(pod) { + hostsMnt, resolvMnt = makeHostNetworkMount(opts) + manifest.Volumes = append(manifest.Volumes, appctypes.Volume{ + Name: convertToACName(hostsMnt.Name), + Kind: "host", + Source: hostsMnt.HostPath, + }) + manifest.Volumes = append(manifest.Volumes, appctypes.Volume{ + Name: convertToACName(resolvMnt.Name), + Kind: "host", + Source: resolvMnt.HostPath, + }) + } + ctx := securitycontext.DetermineEffectiveSecurityContext(pod, &c) if err := setApp(imgManifest, &c, opts, ctx, pod.Spec.SecurityContext); err != nil { return err @@ -751,30 +786,39 @@ func serviceFilePath(serviceName string) string { func (r *Runtime) generateRunCommand(pod *api.Pod, uuid string) (string, error) { runPrepared := r.buildCommand("run-prepared").Args + var hostname string + var err error // Setup network configuration. - if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.HostNetwork { + if kubecontainer.IsHostNetworkPod(pod) { runPrepared = append(runPrepared, "--net=host") + + // TODO(yifan): Let runtimeHelper.GeneratePodHostNameAndDomain() to handle this. + hostname, err = os.Hostname() + if err != nil { + return "", err + } } else { runPrepared = append(runPrepared, fmt.Sprintf("--net=%s", defaultNetworkName)) + + // Setup DNS. + dnsServers, dnsSearches, err := r.runtimeHelper.GetClusterDNS(pod) + if err != nil { + return "", err + } + for _, server := range dnsServers { + runPrepared = append(runPrepared, fmt.Sprintf("--dns=%s", server)) + } + for _, search := range dnsSearches { + runPrepared = append(runPrepared, fmt.Sprintf("--dns-search=%s", search)) + } + if len(dnsServers) > 0 || len(dnsSearches) > 0 { + runPrepared = append(runPrepared, fmt.Sprintf("--dns-opt=%s", defaultDNSOption)) + } + + // TODO(yifan): host domain is not being used. + hostname, _ = r.runtimeHelper.GeneratePodHostNameAndDomain(pod) } - // Setup DNS. - dnsServers, dnsSearches, err := r.runtimeHelper.GetClusterDNS(pod) - if err != nil { - return "", err - } - for _, server := range dnsServers { - runPrepared = append(runPrepared, fmt.Sprintf("--dns=%s", server)) - } - for _, search := range dnsSearches { - runPrepared = append(runPrepared, fmt.Sprintf("--dns-search=%s", search)) - } - if len(dnsServers) > 0 || len(dnsSearches) > 0 { - runPrepared = append(runPrepared, fmt.Sprintf("--dns-opt=%s", defaultDNSOption)) - } - - // TODO(yifan): host domain is not being used. - hostname, _ := r.runtimeHelper.GeneratePodHostNameAndDomain(pod) runPrepared = append(runPrepared, fmt.Sprintf("--hostname=%s", hostname)) runPrepared = append(runPrepared, uuid) return strings.Join(runPrepared, " "), nil diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index 33d9174d478..baebbe1d1a4 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -1039,6 +1039,11 @@ func TestSetApp(t *testing.T) { } func TestGenerateRunCommand(t *testing.T) { + hostName, err := os.Hostname() + if err != nil { + t.Fatalf("Cannot get the hostname: %v", err) + } + tests := []struct { pod *api.Pod uuid string @@ -1094,9 +1099,9 @@ func TestGenerateRunCommand(t *testing.T) { "rkt-uuid-foo", []string{}, []string{}, - "pod-hostname-foo", + "", nil, - "/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=pod-hostname-foo rkt-uuid-foo", + fmt.Sprintf("/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=%s rkt-uuid-foo", hostName), }, // Case #3, returns dns, dns searches, with private-net. { @@ -1117,7 +1122,7 @@ func TestGenerateRunCommand(t *testing.T) { nil, "/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=rkt.kubernetes.io --dns=127.0.0.1 --dns-search=. --dns-opt=ndots:5 --hostname=pod-hostname-foo rkt-uuid-foo", }, - // Case #4, returns dns, dns searches, with host-network. + // Case #4, returns no dns, dns searches, with host-network. { &api.Pod{ ObjectMeta: api.ObjectMeta{ @@ -1134,7 +1139,7 @@ func TestGenerateRunCommand(t *testing.T) { []string{"."}, "pod-hostname-foo", nil, - "/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --dns=127.0.0.1 --dns-search=. --dns-opt=ndots:5 --hostname=pod-hostname-foo rkt-uuid-foo", + fmt.Sprintf("/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=%s rkt-uuid-foo", hostName), }, }