From 351a70b60e5091fd0c6c089091fb5fd122e3a73d Mon Sep 17 00:00:00 2001 From: Filipe Brandenburger Date: Tue, 20 Mar 2018 17:26:48 -0700 Subject: [PATCH 1/2] In summary_test, create a file outside the test volume too. This is necessary to show any RootFs usage on systems where the backing filesystem of overlay2 is xfs. The current test only created directories (for mount points) in the upper layer of the overlay. Outside of the mount namespace, only the directories are visible. When running `du` on those, usually filesystems will show some usage, but not xfs, which shows a disk usage of 0 for directories. Fix this by creating a file in the root directory, outside the volumes, in order to trigger some disk usage that can be measured by `du`. Tested: make test-e2e-node REMOTE=true HOSTS=centos-e2e-node FOCUS="Summary API" --- test/e2e_node/summary_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 5cfdccaffad..80581e17e6e 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -324,7 +324,7 @@ func getSummaryTestPods(f *framework.Framework, numRestarts int32, names ...stri { Name: "busybox-container", Image: busyboxImage, - Command: getRestartingContainerCommand("/test-empty-dir-mnt", 0, numRestarts, "ping -c 1 google.com; echo 'hello world' >> /test-empty-dir-mnt/file;"), + Command: getRestartingContainerCommand("/test-empty-dir-mnt", 0, numRestarts, "echo 'some bytes' >/outside_the_volume.txt; ping -c 1 google.com; echo 'hello world' >> /test-empty-dir-mnt/file;"), Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ // Must set memory limit to get MemoryStats.AvailableBytes From b8c39b705572530deab00cf9ad69837c5970a023 Mon Sep 17 00:00:00 2001 From: Filipe Brandenburger Date: Wed, 21 Mar 2018 08:46:46 -0700 Subject: [PATCH 2/2] In summary_test, make Docker cpu/memory checks optional if unavailable. The numbers will only be available when docker.service has its own memory and cpu cgroups, which doesn't necessarily happen unless the unit has Delegate=yes configured. Let's work around that by checking the status of Delegate, in the case where we are: * running Docker * running Systemd * able to check the status through systemctl * the status is explicitly Delegate=no (the default) If all of those are true, let's make CPU and Memory expectations optional. Tested: make test-e2e-node REMOTE=true HOSTS=centos-e2e-node FOCUS="Summary API" --- test/e2e_node/summary_test.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 80581e17e6e..557d30df9fb 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -19,6 +19,7 @@ package e2e_node import ( "fmt" "io/ioutil" + "os/exec" "strings" "time" @@ -118,9 +119,33 @@ var _ = framework.KubeDescribe("Summary API", func() { "PageFaults": bounded(0, 1000000), "MajorPageFaults": bounded(0, 10), }) + runtimeContExpectations := sysContExpectations().(*gstruct.FieldsMatcher) + if systemdutil.IsRunningSystemd() && framework.TestContext.ContainerRuntime == "docker" { + // Some Linux distributions still ship a docker.service that is missing + // a `Delegate=yes` setting (or equivalent CPUAccounting= and MemoryAccounting=) + // that allows us to monitor the container runtime resource usage through + // the "cpu" and "memory" cgroups. + // + // Make an exception here for those distros, only for Docker, so that they + // can pass the full node e2e tests even in that case. + // + // For newer container runtimes (using CRI) and even distros that still + // ship Docker, we should encourage them to always set `Delegate=yes` in + // order to make monitoring of the runtime possible. + stdout, err := exec.Command("systemctl", "show", "-p", "Delegate", "docker.service").CombinedOutput() + if err == nil && strings.TrimSpace(string(stdout)) == "Delegate=no" { + // Only make these optional if we can successfully confirm that + // Delegate is set to "no" (in other words, unset.) If we fail + // to check that, default to requiring it, which might cause + // false positives, but that should be the safer approach. + By("Making runtime container expectations optional, since systemd was not configured to Delegate=yes the cgroups") + runtimeContExpectations.Fields["Memory"] = Or(BeNil(), runtimeContExpectations.Fields["Memory"]) + runtimeContExpectations.Fields["CPU"] = Or(BeNil(), runtimeContExpectations.Fields["CPU"]) + } + } systemContainers := gstruct.Elements{ "kubelet": sysContExpectations(), - "runtime": sysContExpectations(), + "runtime": runtimeContExpectations, "pods": podsContExpectations, } // The Kubelet only manages the 'misc' system container if the host is not running systemd.