github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 21:47:07 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #22666 from pmorie/pod-ip-flake-redux

Browse Source 
Fix flake in pod IP as env var e2e
This commit is contained in:
Brian Grant 2016-03-11 09:42:40 -08:00
commit c6b4518aed
7 changed files with 40 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/kubelet/container/helpers.go
View File

@ -39,7 +39,7 @@ type HandlerRunner interface {
// RuntimeHelper wraps kubelet to make container runtime // RuntimeHelper wraps kubelet to make container runtime
// able to get necessary informations like the RunContainerOptions, DNS settings. // able to get necessary informations like the RunContainerOptions, DNS settings.
type RuntimeHelper interface { type RuntimeHelper interface {
GenerateRunContainerOptions(pod *api.Pod, container *api.Container) (*RunContainerOptions, error) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*RunContainerOptions, error)
GetClusterDNS(pod *api.Pod) (dnsServers []string, dnsSearches []string, err error) GetClusterDNS(pod *api.Pod) (dnsServers []string, dnsSearches []string, err error)
} }

26
pkg/kubelet/dockertools/manager.go
View File

@ -1491,7 +1491,7 @@ func (dm *DockerManager) applyOOMScoreAdj(container *api.Container, containerInf
// Run a single container from a pod. Returns the docker container ID // Run a single container from a pod. Returns the docker container ID
// If do not need to pass labels, just pass nil. // If do not need to pass labels, just pass nil.
func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Container, netMode, ipcMode, pidMode string, restartCount int) (kubecontainer.ContainerID, error) { func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Container, netMode, ipcMode, pidMode, podIP string, restartCount int) (kubecontainer.ContainerID, error) {
start := time.Now() start := time.Now()
defer func() { defer func() {
metrics.ContainerManagerLatency.WithLabelValues("runContainerInPod").Observe(metrics.SinceInMicroseconds(start)) metrics.ContainerManagerLatency.WithLabelValues("runContainerInPod").Observe(metrics.SinceInMicroseconds(start))
@ -1502,7 +1502,7 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe
glog.Errorf("Can't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) glog.Errorf("Can't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
} }
opts, err := dm.runtimeHelper.GenerateRunContainerOptions(pod, container) opts, err := dm.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
if err != nil { if err != nil {
return kubecontainer.ContainerID{}, fmt.Errorf("GenerateRunContainerOptions: %v", err) return kubecontainer.ContainerID{}, fmt.Errorf("GenerateRunContainerOptions: %v", err)
} }
@ -1635,7 +1635,7 @@ func (dm *DockerManager) createPodInfraContainer(pod *api.Pod) (kubecontainer.Do
} }
// Currently we don't care about restart count of infra container, just set it to 0. // Currently we don't care about restart count of infra container, just set it to 0.
id, err := dm.runContainerInPod(pod, container, netNamespace, getIPCMode(pod), getPidMode(pod), 0) id, err := dm.runContainerInPod(pod, container, netNamespace, getIPCMode(pod), getPidMode(pod), "", 0)
if err != nil { if err != nil {
return "", kubecontainer.ErrRunContainer, err.Error() return "", kubecontainer.ErrRunContainer, err.Error()
} }
@ -1832,6 +1832,19 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec
} }
} }
// We pass the value of the podIP down to runContainerInPod, which in turn
// passes it to various other functions, in order to facilitate
// functionality that requires this value (hosts file and downward API)
// and avoid races determining the pod IP in cases where a container
// requires restart but the podIP isn't in the status manager yet.
//
// We default to the IP in the passed-in pod status, and overwrite it if the
// infra container needs to be (re)started.
podIP := ""
if podStatus != nil {
podIP = podStatus.IP
}
// If we should create infra container then we do it first. // If we should create infra container then we do it first.
podInfraContainerID := containerChanges.InfraContainerId podInfraContainerID := containerChanges.InfraContainerId
if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) { if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) {
@ -1884,9 +1897,8 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec
} }
} }
// Find the pod IP after starting the infra container in order to expose // Overwrite the podIP passed in the pod status, since we just started the infra container.
// it safely via the downward API without a race and be able to use podIP in kubelet-managed /etc/hosts file. podIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer)
pod.Status.PodIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer)
} }
} }
@ -1934,7 +1946,7 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec
// and IPC namespace. PID mode cannot point to another container right now. // and IPC namespace. PID mode cannot point to another container right now.
// See createPodInfraContainer for infra container setup. // See createPodInfraContainer for infra container setup.
namespaceMode := fmt.Sprintf("container:%v", podInfraContainerID) namespaceMode := fmt.Sprintf("container:%v", podInfraContainerID)
_, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, getPidMode(pod), restartCount) _, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, getPidMode(pod), podIP, restartCount)
if err != nil { if err != nil {
startContainerResult.Fail(kubecontainer.ErrRunContainer, err.Error()) startContainerResult.Fail(kubecontainer.ErrRunContainer, err.Error())
// TODO(bburns) : Perhaps blacklist a container after N failures? // TODO(bburns) : Perhaps blacklist a container after N failures?

2
pkg/kubelet/dockertools/manager_test.go
View File

@ -67,7 +67,7 @@ var _ kubecontainer.RuntimeHelper = &fakeRuntimeHelper{}
var testPodContainerDir string var testPodContainerDir string
func (f *fakeRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container) (*kubecontainer.RunContainerOptions, error) { func (f *fakeRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
var opts kubecontainer.RunContainerOptions var opts kubecontainer.RunContainerOptions
var err error var err error
if len(container.TerminationMessagePath) != 0 { if len(container.TerminationMessagePath) != 0 {

22
pkg/kubelet/kubelet.go
View File

@ -1225,14 +1225,14 @@ func (kl *Kubelet) relabelVolumes(pod *api.Pod, volumes kubecontainer.VolumeMap)
return nil return nil
} }
func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, hostDomain string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) { func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) {
// Kubernetes only mounts on /etc/hosts if : // Kubernetes only mounts on /etc/hosts if :
// - container does not use hostNetwork and // - container does not use hostNetwork and
// - container is not a infrastructure(pause) container // - container is not a infrastructure(pause) container
// - container is not already mounting on /etc/hosts // - container is not already mounting on /etc/hosts
// When the pause container is being created, its IP is still unknown. Hence, PodIP will not have been set. // When the pause container is being created, its IP is still unknown. Hence, PodIP will not have been set.
mountEtcHostsFile := (pod.Spec.SecurityContext == nil || !pod.Spec.SecurityContext.HostNetwork) && len(pod.Status.PodIP) > 0 mountEtcHostsFile := (pod.Spec.SecurityContext == nil || !pod.Spec.SecurityContext.HostNetwork) && len(podIP) > 0
glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, pod.Status.PodIP, mountEtcHostsFile) glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile)
mounts := []kubecontainer.Mount{} mounts := []kubecontainer.Mount{}
for _, mount := range container.VolumeMounts { for _, mount := range container.VolumeMounts {
mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath) mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath)
@ -1259,7 +1259,7 @@ func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName,
}) })
} }
if mountEtcHostsFile { if mountEtcHostsFile {
hostsMount, err := makeHostsMount(podDir, pod.Status.PodIP, hostName, hostDomain) hostsMount, err := makeHostsMount(podDir, podIP, hostName, hostDomain)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1361,7 +1361,7 @@ func generatePodHostNameAndDomain(pod *api.Pod, clusterDomain string) (string, s
// GenerateRunContainerOptions generates the RunContainerOptions, which can be used by // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by
// the container runtime to set parameters for launching a container. // the container runtime to set parameters for launching a container.
func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Container) (*kubecontainer.RunContainerOptions, error) { func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
var err error var err error
opts := &kubecontainer.RunContainerOptions{CgroupParent: kl.cgroupRoot} opts := &kubecontainer.RunContainerOptions{CgroupParent: kl.cgroupRoot}
hostname, hostDomainName := generatePodHostNameAndDomain(pod, kl.clusterDomain) hostname, hostDomainName := generatePodHostNameAndDomain(pod, kl.clusterDomain)
@ -1382,11 +1382,11 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont
} }
} }
opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, vol) opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, vol)
if err != nil { if err != nil {
return nil, err return nil, err
} }
opts.Envs, err = kl.makeEnvironmentVariables(pod, container) opts.Envs, err = kl.makeEnvironmentVariables(pod, container, podIP)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1463,7 +1463,7 @@ func (kl *Kubelet) getServiceEnvVarMap(ns string) (map[string]string, error) {
} }
// Make the service environment variables for a pod in the given namespace. // Make the service environment variables for a pod in the given namespace.
func (kl *Kubelet) makeEnvironmentVariables(pod *api.Pod, container *api.Container) ([]kubecontainer.EnvVar, error) { func (kl *Kubelet) makeEnvironmentVariables(pod *api.Pod, container *api.Container, podIP string) ([]kubecontainer.EnvVar, error) {
var result []kubecontainer.EnvVar var result []kubecontainer.EnvVar
// Note: These are added to the docker.Config, but are not included in the checksum computed // Note: These are added to the docker.Config, but are not included in the checksum computed
// by dockertools.BuildDockerName(...). That way, we can still determine whether an // by dockertools.BuildDockerName(...). That way, we can still determine whether an
@ -1510,7 +1510,7 @@ func (kl *Kubelet) makeEnvironmentVariables(pod *api.Pod, container *api.Contain
// Step 1b: resolve alternate env var sources // Step 1b: resolve alternate env var sources
switch { switch {
case envVar.ValueFrom.FieldRef != nil: case envVar.ValueFrom.FieldRef != nil:
runtimeVal, err = kl.podFieldSelectorRuntimeValue(envVar.ValueFrom.FieldRef, pod) runtimeVal, err = kl.podFieldSelectorRuntimeValue(envVar.ValueFrom.FieldRef, pod, podIP)
if err != nil { if err != nil {
return result, err return result, err
} }
@ -1557,14 +1557,14 @@ func (kl *Kubelet) makeEnvironmentVariables(pod *api.Pod, container *api.Contain
return result, nil return result, nil
} }
func (kl *Kubelet) podFieldSelectorRuntimeValue(fs *api.ObjectFieldSelector, pod *api.Pod) (string, error) { func (kl *Kubelet) podFieldSelectorRuntimeValue(fs *api.ObjectFieldSelector, pod *api.Pod, podIP string) (string, error) {
internalFieldPath, _, err := api.Scheme.ConvertFieldLabel(fs.APIVersion, "Pod", fs.FieldPath, "") internalFieldPath, _, err := api.Scheme.ConvertFieldLabel(fs.APIVersion, "Pod", fs.FieldPath, "")
if err != nil { if err != nil {
return "", err return "", err
} }
switch internalFieldPath { switch internalFieldPath {
case "status.podIP": case "status.podIP":
return pod.Status.PodIP, nil return podIP, nil
} }
return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath) return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath)
} }

10
pkg/kubelet/kubelet_test.go
View File

@ -728,7 +728,7 @@ func TestMakeVolumeMounts(t *testing.T) {
}, },
} }
mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", podVolumes) mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes)
expectedMounts := []kubecontainer.Mount{ expectedMounts := []kubecontainer.Mount{
{ {
@ -1189,7 +1189,7 @@ func TestDNSConfigurationParams(t *testing.T) {
for i, pod := range pods { for i, pod := range pods {
var err error var err error
kubelet.volumeManager.SetVolumes(pod.UID, make(kubecontainer.VolumeMap, 0)) kubelet.volumeManager.SetVolumes(pod.UID, make(kubecontainer.VolumeMap, 0))
options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}) options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}, "")
if err != nil { if err != nil {
t.Fatalf("failed to generate container options: %v", err) t.Fatalf("failed to generate container options: %v", err)
} }
@ -1210,7 +1210,7 @@ func TestDNSConfigurationParams(t *testing.T) {
kubelet.resolverConfig = "/etc/resolv.conf" kubelet.resolverConfig = "/etc/resolv.conf"
for i, pod := range pods { for i, pod := range pods {
var err error var err error
options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}) options[i], err = kubelet.GenerateRunContainerOptions(pod, &api.Container{}, "")
if err != nil { if err != nil {
t.Fatalf("failed to generate container options: %v", err) t.Fatalf("failed to generate container options: %v", err)
} }
@ -1715,9 +1715,9 @@ func TestMakeEnvironmentVariables(t *testing.T) {
Name: "dapi-test-pod-name", Name: "dapi-test-pod-name",
}, },
} }
testPod.Status.PodIP = "1.2.3.4" podIP := "1.2.3.4"
result, err := kl.makeEnvironmentVariables(testPod, tc.container) result, err := kl.makeEnvironmentVariables(testPod, tc.container, podIP)
if err != nil { if err != nil {
t.Errorf("[%v] Unexpected error: %v", tc.name, err) t.Errorf("[%v] Unexpected error: %v", tc.name, err)
} }

2
pkg/kubelet/rkt/fake_rkt_interface_test.go
View File

@ -153,7 +153,7 @@ type fakeRuntimeHelper struct {
err error err error
} }
func (f *fakeRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container) (*kubecontainer.RunContainerOptions, error) { func (f *fakeRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
return nil, fmt.Errorf("Not implemented") return nil, fmt.Errorf("Not implemented")
} }

3
pkg/kubelet/rkt/rkt.go
View File

@ -579,7 +579,8 @@ func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, c api.Container, pullSecrets [
return err return err
} }
opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c) // TODO: determine how this should be handled for rkt
opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, "")
if err != nil { if err != nil {
return err return err
} }