diff --git a/Makefile b/Makefile index af0950c6b1e..69f4a611d07 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-e2e: ginkgo generated_files # DELETE_INSTANCES: For REMOTE=true only. Delete any instances created as # part of this test run. Defaults to false. # ARTIFACTS: For REMOTE=true only. Local directory to scp test artifacts into -# from the remote hosts. Defaults to "/tmp/_artifacts". +# from the remote hosts. Defaults to ""/tmp/_artifacts". # REPORT: For REMOTE=false only. Local directory to write juntil xml results # to. Defaults to "/tmp/". # CLEANUP: For REMOTE=true only. If false, do not stop processes or delete diff --git a/docs/design/resource-qos.md b/docs/design/resource-qos.md index 6a8e8ab28a4..b2a43c97937 100644 --- a/docs/design/resource-qos.md +++ b/docs/design/resource-qos.md @@ -226,7 +226,6 @@ Pod OOM score configuration *Pod infra containers* or *Special Pod init process* - OOM_SCORE_ADJ: -998 - *Kubelet, Docker* - OOM_SCORE_ADJ: -999 (won’t be OOM killed) - Hack, because these critical tasks might die if they conflict with guaranteed containers. In the future, we should place all user-pods into a separate cgroup, and set a limit on the memory they can consume. diff --git a/docs/proposals/kubelet-systemd.md b/docs/proposals/kubelet-systemd.md index 1b7c0373e06..d7c42cf6670 100644 --- a/docs/proposals/kubelet-systemd.md +++ b/docs/proposals/kubelet-systemd.md @@ -365,7 +365,7 @@ The `kubelet` will set the following: The `kubelet` at bootstrapping will set the `oom_score_adj` value for Kubernetes daemons, and any dependent container-runtime daemons. -If `container-runtime` is set to `docker`, then set its `oom_score_adj=-999` +If `container-runtime` is set to `docker`, then set its `oom_score_adj=-900` ## Implementation concerns diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index c284e88a4ef..5601b95c846 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -100,8 +100,6 @@ pkg/kubelet/api/v1alpha1/runtime/api.pb.go: ContainerPort *int32 `protobuf:"vari pkg/kubelet/api/v1alpha1/runtime/api.pb.go: OomScoreAdj *int64 `protobuf:"varint,5,opt,name=oom_score_adj,json=oomScoreAdj" json:"oom_score_adj,omitempty"` pkg/kubelet/api/v1alpha1/runtime/api.proto: optional int32 container_port = 3; pkg/kubelet/api/v1alpha1/runtime/api.proto: optional int64 oom_score_adj = 5; -pkg/kubelet/cm/container_manager_linux.go: glog.V(3).Infof("Failed to apply oom_score_adj %d for pid %d: %v", oomScoreAdj, pid, err) -pkg/kubelet/cm/container_manager_linux.go: glog.V(5).Infof("attempting to apply oom_score_adj of %d to pid %d", oomScoreAdj, pid) pkg/kubelet/network/hairpin/hairpin.go: hairpinModeRelativePath = "hairpin_mode" pkg/kubelet/qos/policy_test.go: t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOOMScoreAdj, test.highOOMScoreAdj, oomScoreAdj) pkg/kubelet/qos/policy_test.go: highOOMScoreAdj int // The min oom_score_adj score the container should be assigned. @@ -119,12 +117,6 @@ test/e2e/common/host_path.go: fmt.Sprintf("--retry_time=%d", retryDuration), test/e2e/es_cluster_logging.go: framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse) test/e2e/es_cluster_logging.go: // Check to see if have a cluster_name field. test/e2e/es_cluster_logging.go: clusterName, ok := esResponse["cluster_name"] -test/e2e_node/container_manager_test.go: return fmt.Errorf("expected pid %d's oom_score_adj to be %d; found %d", pid, expectedOOMScoreAdj, oomScore) -test/e2e_node/container_manager_test.go: return fmt.Errorf("expected pid %d's oom_score_adj to be < %d; found %d", pid, expectedMaxOOMScoreAdj, oomScore) -test/e2e_node/container_manager_test.go: return fmt.Errorf("expected pid %d's oom_score_adj to be >= %d; found %d", pid, expectedMinOOMScoreAdj, oomScore) -test/e2e_node/container_manager_test.go: return fmt.Errorf("failed to get oom_score_adj for %d", pid) -test/e2e_node/container_manager_test.go: return fmt.Errorf("failed to get oom_score_adj for %d: %v", pid, err) -test/e2e_node/container_manager_test.go: procfsPath := path.Join("/proc", strconv.Itoa(pid), "oom_score_adj") test/images/mount-tester/mt.go: flag.BoolVar(&breakOnExpectedContent, "break_on_expected_content", true, "Break out of loop on expected content, (use with --file_content_in_loop flag only)") test/images/mount-tester/mt.go: flag.IntVar(&retryDuration, "retry_time", 180, "Retry time during the loop") test/images/mount-tester/mt.go: flag.StringVar(&readFileContentInLoopPath, "file_content_in_loop", "", "Path to read the file content in loop from") diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 7abcfdbf46b..22a6c99f793 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -326,7 +326,6 @@ func (cm *containerManagerImpl) setupNode() error { systemContainers := []*systemContainer{} if cm.ContainerRuntime == "docker" { - dockerVersion := getDockerVersion(cm.cadvisorInterface) if cm.RuntimeCgroupsName != "" { cont := newSystemCgroups(cm.RuntimeCgroupsName) var capacity = api.ResourceList{} @@ -352,17 +351,13 @@ func (cm *containerManagerImpl) setupNode() error { }, }, } + dockerVersion := getDockerVersion(cm.cadvisorInterface) cont.ensureStateFunc = func(manager *fs.Manager) error { - return ensureDockerInContainer(dockerVersion, qos.DockerOOMScoreAdj, dockerContainer) + return ensureDockerInContainer(dockerVersion, -900, dockerContainer) } systemContainers = append(systemContainers, cont) } else { cm.periodicTasks = append(cm.periodicTasks, func() { - glog.V(10).Infof("Adding docker daemon periodic tasks") - if err := ensureDockerInContainer(dockerVersion, qos.DockerOOMScoreAdj, nil); err != nil { - glog.Error(err) - return - } cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile) if err != nil { glog.Error(err) @@ -406,15 +401,11 @@ func (cm *containerManagerImpl) setupNode() error { }, } cont.ensureStateFunc = func(_ *fs.Manager) error { - return ensureProcessInContainerWithOOMScore(os.Getpid(), qos.KubeletOOMScoreAdj, &manager) + return manager.Apply(os.Getpid()) } systemContainers = append(systemContainers, cont) } else { cm.periodicTasks = append(cm.periodicTasks, func() { - if err := ensureProcessInContainerWithOOMScore(os.Getpid(), qos.KubeletOOMScoreAdj, nil); err != nil { - glog.Error(err) - return - } cont, err := getContainer(os.Getpid()) if err != nil { glog.Errorf("failed to find cgroups of kubelet - %v", err) @@ -525,18 +516,16 @@ func (cm *containerManagerImpl) SystemCgroupsLimit() api.ResourceList { } func isProcessRunningInHost(pid int) (bool, error) { - // Get init pid namespace. - initPidNs, err := os.Readlink("/proc/1/ns/pid") + // Get init mount namespace. Mount namespace is unique for all containers. + initMntNs, err := os.Readlink("/proc/1/ns/mnt") if err != nil { - return false, fmt.Errorf("failed to find pid namespace of init process") + return false, fmt.Errorf("failed to find mount namespace of init process") } - glog.V(10).Infof("init pid ns is %q", initPidNs) - processPidNs, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", pid)) + processMntNs, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/mnt", pid)) if err != nil { - return false, fmt.Errorf("failed to find pid namespace of process %q", pid) + return false, fmt.Errorf("failed to find mount namespace of process %q", pid) } - glog.V(10).Infof("Pid %d pid ns is %q", pid, processPidNs) - return initPidNs == processPidNs, nil + return initMntNs == processMntNs, nil } func getPidFromPidFile(pidFile string) (int, error) { @@ -578,6 +567,7 @@ func ensureDockerInContainer(dockerVersion semver.Version, oomScoreAdj int, mana if dockerVersion.GTE(containerdVersion) { dockerProcs = append(dockerProcs, process{containerdProcessName, containerdPidFile}) } + var errs []error for _, proc := range dockerProcs { pids, err := getPidsForProcess(proc.name, proc.file) @@ -588,7 +578,7 @@ func ensureDockerInContainer(dockerVersion semver.Version, oomScoreAdj int, mana // Move if the pid is not already in the desired container. for _, pid := range pids { - if err := ensureProcessInContainerWithOOMScore(pid, oomScoreAdj, manager); err != nil { + if err := ensureProcessInContainer(pid, oomScoreAdj, manager); err != nil { errs = append(errs, fmt.Errorf("errors moving %q pid: %v", proc.name, err)) } } @@ -596,37 +586,32 @@ func ensureDockerInContainer(dockerVersion semver.Version, oomScoreAdj int, mana return utilerrors.NewAggregate(errs) } -func ensureProcessInContainerWithOOMScore(pid int, oomScoreAdj int, manager *fs.Manager) error { +func ensureProcessInContainer(pid int, oomScoreAdj int, manager *fs.Manager) error { if runningInHost, err := isProcessRunningInHost(pid); err != nil { // Err on the side of caution. Avoid moving the docker daemon unless we are able to identify its context. return err } else if !runningInHost { // Process is running inside a container. Don't touch that. - glog.V(2).Infof("pid %d is not running in the host namespaces", pid) return nil } var errs []error - if manager != nil { - cont, err := getContainer(pid) - if err != nil { - errs = append(errs, fmt.Errorf("failed to find container of PID %d: %v", pid, err)) - } + cont, err := getContainer(pid) + if err != nil { + errs = append(errs, fmt.Errorf("failed to find container of PID %d: %v", pid, err)) + } - if cont != manager.Cgroups.Name { - err = manager.Apply(pid) - if err != nil { - errs = append(errs, fmt.Errorf("failed to move PID %d (in %q) to %q: %v", pid, cont, manager.Cgroups.Name, err)) - } + if cont != manager.Cgroups.Name { + err = manager.Apply(pid) + if err != nil { + errs = append(errs, fmt.Errorf("failed to move PID %d (in %q) to %q", pid, cont, manager.Cgroups.Name)) } } // Also apply oom-score-adj to processes oomAdjuster := oom.NewOOMAdjuster() - glog.V(5).Infof("attempting to apply oom_score_adj of %d to pid %d", oomScoreAdj, pid) if err := oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err != nil { - glog.V(3).Infof("Failed to apply oom_score_adj %d for pid %d: %v", oomScoreAdj, pid, err) - errs = append(errs, fmt.Errorf("failed to apply oom score %d to PID %d: %v", oomScoreAdj, pid, err)) + errs = append(errs, fmt.Errorf("failed to apply oom score %d to PID %d", oomScoreAdj, pid)) } return utilerrors.NewAggregate(errs) } diff --git a/pkg/kubelet/qos/policy.go b/pkg/kubelet/qos/policy.go index 7013f712f87..ad696f36109 100644 --- a/pkg/kubelet/qos/policy.go +++ b/pkg/kubelet/qos/policy.go @@ -21,9 +21,8 @@ import ( ) const ( - PodInfraOOMAdj int = -998 + PodInfraOOMAdj int = -999 KubeletOOMScoreAdj int = -999 - DockerOOMScoreAdj int = -999 KubeProxyOOMScoreAdj int = -999 guaranteedOOMScoreAdj int = -998 besteffortOOMScoreAdj int = 1000 @@ -54,10 +53,10 @@ func GetContainerOOMScoreAdjust(pod *api.Pod, container *api.Container, memoryCa // Note that this is a heuristic, it won't work if a container has many small processes. memoryRequest := container.Resources.Requests.Memory().Value() oomScoreAdjust := 1000 - (1000*memoryRequest)/memoryCapacity - // A guaranteed pod using 100% of memory can have an OOM score of 10. Ensure + // A guaranteed pod using 100% of memory can have an OOM score of 1. Ensure // that burstable pods have a higher OOM score adjustment. - if int(oomScoreAdjust) < (1000 + guaranteedOOMScoreAdj) { - return (1000 + guaranteedOOMScoreAdj) + if oomScoreAdjust < 2 { + return 2 } // Give burstable pods a higher chance of survival over besteffort pods. if int(oomScoreAdjust) == besteffortOOMScoreAdj { diff --git a/pkg/util/oom/oom_linux.go b/pkg/util/oom/oom_linux.go index 6b9c93cf170..0bb8703874e 100644 --- a/pkg/util/oom/oom_linux.go +++ b/pkg/util/oom/oom_linux.go @@ -20,7 +20,6 @@ package oom import ( "fmt" - "io/ioutil" "os" "path" "strconv" @@ -66,24 +65,28 @@ func applyOOMScoreAdj(pid int, oomScoreAdj int) error { maxTries := 2 oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj") value := strconv.Itoa(oomScoreAdj) - glog.V(4).Infof("attempting to set %q to %q", oomScoreAdjPath, value) var err error for i := 0; i < maxTries; i++ { - err = ioutil.WriteFile(oomScoreAdjPath, []byte(value), 0700) + f, err := os.Open(oomScoreAdjPath) if err != nil { if os.IsNotExist(err) { - glog.V(2).Infof("%q does not exist", oomScoreAdjPath) return os.ErrNotExist } - - glog.V(3).Info(err) + err = fmt.Errorf("failed to apply oom-score-adj to pid %d (%v)", pid, err) + continue + } + if _, err := f.Write([]byte(value)); err != nil { + // we can ignore the return value of f.Close() here. + f.Close() + err = fmt.Errorf("failed to apply oom-score-adj to pid %d (%v)", pid, err) + continue + } + if err = f.Close(); err != nil { + err = fmt.Errorf("failed to apply oom-score-adj to pid %d (%v)", pid, err) continue } return nil } - if err != nil { - glog.V(2).Infof("failed to set %q to %q: %v", oomScoreAdjPath, value, err) - } return err } diff --git a/test/e2e_node/container_manager_test.go b/test/e2e_node/container_manager_test.go index e5fd987c485..ac8e659e02c 100644 --- a/test/e2e_node/container_manager_test.go +++ b/test/e2e_node/container_manager_test.go @@ -18,14 +18,9 @@ package e2e_node import ( "fmt" - "os/exec" - "path" - "strconv" - "strings" "time" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" @@ -33,193 +28,63 @@ import ( . "github.com/onsi/gomega" ) -const ( - kubeletProcessname = "kubelet" -) - -func getOOMScoreForPid(pid int) (int, error) { - procfsPath := path.Join("/proc", strconv.Itoa(pid), "oom_score_adj") - out, err := exec.Command("sudo", "cat", procfsPath).CombinedOutput() - if err != nil { - return 0, err - } - return strconv.Atoi(strings.TrimSpace(string(out))) -} - -func validateOOMScoreAdjSetting(pid int, expectedOOMScoreAdj int) error { - oomScore, err := getOOMScoreForPid(pid) - if err != nil { - return fmt.Errorf("failed to get oom_score_adj for %d: %v", pid, err) - } - if expectedOOMScoreAdj != oomScore { - return fmt.Errorf("expected pid %d's oom_score_adj to be %d; found %d", pid, expectedOOMScoreAdj, oomScore) - } - return nil -} - -func validateOOMScoreAdjSettingIsInRange(pid int, expectedMinOOMScoreAdj, expectedMaxOOMScoreAdj int) error { - oomScore, err := getOOMScoreForPid(pid) - if err != nil { - return fmt.Errorf("failed to get oom_score_adj for %d", pid) - } - if oomScore < expectedMinOOMScoreAdj { - return fmt.Errorf("expected pid %d's oom_score_adj to be >= %d; found %d", pid, expectedMinOOMScoreAdj, oomScore) - } - if oomScore < expectedMaxOOMScoreAdj { - return fmt.Errorf("expected pid %d's oom_score_adj to be < %d; found %d", pid, expectedMaxOOMScoreAdj, oomScore) - } - return nil -} - -var _ = framework.KubeDescribe("Kubelet Container Manager [Serial]", func() { +var _ = framework.KubeDescribe("Kubelet Container Manager", func() { f := framework.NewDefaultFramework("kubelet-container-manager") + var podClient *framework.PodClient - Describe("Validate OOM score adjustments", func() { - Context("once the node is setup", func() { - It("docker daemon's oom-score-adj should be -999", func() { - dockerPids, err := getPidsForProcess(dockerProcessName, dockerPidFile) - Expect(err).To(BeNil(), "failed to get list of docker daemon pids") - for _, pid := range dockerPids { - Eventually(func() error { - return validateOOMScoreAdjSetting(pid, -999) - }, 5*time.Minute, 30*time.Second).Should(BeNil()) - } - }) - It("Kubelet's oom-score-adj should be -999", func() { - kubeletPids, err := getPidsForProcess(kubeletProcessName, "") - Expect(err).To(BeNil(), "failed to get list of kubelet pids") - Expect(len(kubeletPids)).To(Equal(1), "expected only one kubelet process; found %d", len(kubeletPids)) - Eventually(func() error { - return validateOOMScoreAdjSetting(kubeletPids[0], -999) - }, 5*time.Minute, 30*time.Second).Should(BeNil()) - }) - It("pod infra containers oom-score-adj should be -998 and best effort container's should be 1000", func() { - var err error - podClient := f.PodClient() - podName := "besteffort" + string(uuid.NewUUID()) + BeforeEach(func() { + podClient = f.PodClient() + }) + + Describe("oom score adjusting", func() { + Context("when scheduling a busybox command that always fails in a pod", func() { + var podName string + + BeforeEach(func() { + podName = "bin-false" + string(uuid.NewUUID()) podClient.Create(&api.Pod{ ObjectMeta: api.ObjectMeta{ Name: podName, }, Spec: api.PodSpec{ + // Don't restart the Pod since it is expected to exit + RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[serveHostnameImage], - Name: podName, + Image: ImageRegistry[busyBoxImage], + Name: podName, + Command: []string{"/bin/false"}, }, }, }, }) - var pausePids []int - By("checking infra container's oom-score-adj") - Eventually(func() error { - pausePids, err = getPidsForProcess("pause", "") - if err != nil { - return fmt.Errorf("failed to get list of pause pids: %v", err) - } - for _, pid := range pausePids { - if err := validateOOMScoreAdjSetting(pid, -998); err != nil { - return err - } - } - return nil - }, 2*time.Minute, time.Second*4).Should(BeNil()) - var shPids []int - By("checking besteffort container's oom-score-adj") - Eventually(func() error { - shPids, err = getPidsForProcess("serve_hostname", "") - if err != nil { - return fmt.Errorf("failed to get list of serve hostname process pids: %v", err) - } - if len(shPids) != 1 { - return fmt.Errorf("expected only one serve_hostname process; found %d", len(shPids)) - } - return validateOOMScoreAdjSetting(shPids[0], 1000) - }, 2*time.Minute, time.Second*4).Should(BeNil()) - }) - It("guaranteed container's oom-score-adj should be -998", func() { - podClient := f.PodClient() - podName := "guaranteed" + string(uuid.NewUUID()) - podClient.Create(&api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: podName, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Image: ImageRegistry[nginxImage], - Name: podName, - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("50Mi"), - }, - }, - }, - }, - }, - }) - var ( - ngPids []int - err error - ) + + It("should have an error terminated reason", func() { Eventually(func() error { - ngPids, err = getPidsForProcess("nginx", "") + podData, err := podClient.Get(podName) if err != nil { - return fmt.Errorf("failed to get list of nginx process pids: %v", err) + return err } - for _, pid := range ngPids { - if err := validateOOMScoreAdjSetting(pid, -998); err != nil { - return err - } + if len(podData.Status.ContainerStatuses) != 1 { + return fmt.Errorf("expected only one container in the pod %q", podName) + } + contTerminatedState := podData.Status.ContainerStatuses[0].State.Terminated + if contTerminatedState == nil { + return fmt.Errorf("expected state to be terminated. Got pod status: %+v", podData.Status) + } + if contTerminatedState.Reason != "Error" { + return fmt.Errorf("expected terminated state reason to be error. Got %+v", contTerminatedState) } - return nil - }, 2*time.Minute, time.Second*4).Should(BeNil()) - + }, time.Minute, time.Second*4).Should(BeNil()) }) - It("burstable container's oom-score-adj should be between [2, 1000)", func() { - podClient := f.PodClient() - podName := "burstable" + string(uuid.NewUUID()) - podClient.Create(&api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: podName, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Image: ImageRegistry[testWebServer], - Name: podName, - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("50Mi"), - }, - }, - }, - }, - }, - }) - var ( - wsPids []int - err error - ) - Eventually(func() error { - wsPids, err = getPidsForProcess("test-webserver", "") - if err != nil { - return fmt.Errorf("failed to get list of test-webserver process pids: %v", err) - } - for _, pid := range wsPids { - if err := validateOOMScoreAdjSettingIsInRange(pid, 2, 1000); err != nil { - return err - } - } - return nil - }, 2*time.Minute, time.Second*4).Should(BeNil()) - // TODO: Test the oom-score-adj logic for burstable more accurately. + It("should be possible to delete", func() { + err := podClient.Delete(podName, &api.DeleteOptions{}) + Expect(err).To(BeNil(), fmt.Sprintf("Error deleting Pod %v", err)) }) }) }) + }) diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index 445dc52196b..6f8657e9315 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -75,54 +75,7 @@ var _ = framework.KubeDescribe("Kubelet", func() { }, time.Minute, time.Second*4).Should(Equal("Hello World\n")) }) }) - Context("when scheduling a busybox command that always fails in a pod", func() { - var podName string - BeforeEach(func() { - podName = "bin-false" + string(uuid.NewUUID()) - podClient.Create(&api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: podName, - }, - Spec: api.PodSpec{ - // Don't restart the Pod since it is expected to exit - RestartPolicy: api.RestartPolicyNever, - Containers: []api.Container{ - { - Image: ImageRegistry[busyBoxImage], - Name: podName, - Command: []string{"/bin/false"}, - }, - }, - }, - }) - }) - - It("should have an error terminated reason", func() { - Eventually(func() error { - podData, err := podClient.Get(podName) - if err != nil { - return err - } - if len(podData.Status.ContainerStatuses) != 1 { - return fmt.Errorf("expected only one container in the pod %q", podName) - } - contTerminatedState := podData.Status.ContainerStatuses[0].State.Terminated - if contTerminatedState == nil { - return fmt.Errorf("expected state to be terminated. Got pod status: %+v", podData.Status) - } - if contTerminatedState.Reason != "Error" { - return fmt.Errorf("expected terminated state reason to be error. Got %+v", contTerminatedState) - } - return nil - }, time.Minute, time.Second*4).Should(BeNil()) - }) - - It("should be possible to delete", func() { - err := podClient.Delete(podName, &api.DeleteOptions{}) - Expect(err).To(BeNil(), fmt.Sprintf("Error deleting Pod %v", err)) - }) - }) Context("when scheduling a read only busybox container", func() { podName := "busybox-readonly-fs" + string(uuid.NewUUID()) It("it should not write to root filesystem", func() { diff --git a/test/e2e_node/resource_collector.go b/test/e2e_node/resource_collector.go index 0ccdfc1fd29..f7498c6e379 100644 --- a/test/e2e_node/resource_collector.go +++ b/test/e2e_node/resource_collector.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "log" "os" + "os/exec" "sort" "strconv" "strings" @@ -37,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/util/procfs" "k8s.io/kubernetes/pkg/util/runtime" "k8s.io/kubernetes/pkg/util/uuid" "k8s.io/kubernetes/pkg/util/wait" @@ -450,16 +450,19 @@ const ( containerdPidFile = "/run/docker/libcontainerd/docker-containerd.pid" ) -func getPidsForProcess(name, pidFile string) ([]int, error) { - if len(pidFile) > 0 { - if pid, err := getPidFromPidFile(pidFile); err == nil { - return []int{pid}, nil - } else { - // log the error and fall back to pidof - runtime.HandleError(err) - } +func getContainerNameForProcess(name, pidFile string) (string, error) { + pids, err := getPidsForProcess(name, pidFile) + if err != nil { + return "", fmt.Errorf("failed to detect process id for %q - %v", name, err) } - return procfs.PidOf(name) + if len(pids) == 0 { + return "", nil + } + cont, err := getContainer(pids[0]) + if err != nil { + return "", err + } + return cont, nil } func getPidFromPidFile(pidFile string) (int, error) { @@ -482,19 +485,31 @@ func getPidFromPidFile(pidFile string) (int, error) { return pid, nil } -func getContainerNameForProcess(name, pidFile string) (string, error) { - pids, err := getPidsForProcess(name, pidFile) +func getPidsForProcess(name, pidFile string) ([]int, error) { + if len(pidFile) > 0 { + if pid, err := getPidFromPidFile(pidFile); err == nil { + return []int{pid}, nil + } else { + // log the error and fall back to pidof + runtime.HandleError(err) + } + } + + out, err := exec.Command("pidof", name).Output() if err != nil { - return "", fmt.Errorf("failed to detect process id for %q - %v", name, err) + return []int{}, fmt.Errorf("failed to find pid of %q: %v", name, err) } - if len(pids) == 0 { - return "", nil + + // The output of pidof is a list of pids. + pids := []int{} + for _, pidStr := range strings.Split(strings.TrimSpace(string(out)), " ") { + pid, err := strconv.Atoi(pidStr) + if err != nil { + continue + } + pids = append(pids, pid) } - cont, err := getContainer(pids[0]) - if err != nil { - return "", err - } - return cont, nil + return pids, nil } // getContainer returns the cgroup associated with the specified pid.