diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 230d8dbebc9..384714fc4e6 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -19,6 +19,8 @@ go_library( "node_problem_detector_linux.go", "resource_collector.go", "util.go", + "util_xfs_linux.go", + "util_xfs_unsupported.go", ], importpath = "k8s.io/kubernetes/test/e2e_node", visibility = ["//visibility:public"], @@ -41,7 +43,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", @@ -62,6 +66,7 @@ go_library( "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/util/mount:go_default_library", "//pkg/util/procfs:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -105,6 +110,7 @@ go_test( "node_perf_test.go", "pids_test.go", "pods_container_manager_test.go", + "quota_lsci_test.go", "resource_metrics_test.go", "resource_usage_test.go", "restart_test.go", @@ -138,6 +144,8 @@ go_test( "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/security/apparmor:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/volume/util/quota:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index 34c548029b1..416bf509ea4 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -49,6 +49,7 @@ var NodeImageWhiteList = sets.NewString( busyboxImage, "k8s.gcr.io/busybox@sha256:4bdd623e848417d96127e16037743f0cd8b528c026e9175e22a84f639eca58ff", imageutils.GetE2EImage(imageutils.Nginx), + imageutils.GetE2EImage(imageutils.Perl), imageutils.GetE2EImage(imageutils.ServeHostname), imageutils.GetE2EImage(imageutils.Netexec), imageutils.GetE2EImage(imageutils.Nonewprivs), diff --git a/test/e2e_node/quota_lsci_test.go b/test/e2e_node/quota_lsci_test.go new file mode 100644 index 00000000000..2fc09ff4053 --- /dev/null +++ b/test/e2e_node/quota_lsci_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "path/filepath" + "time" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/quota" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" +) + +const ( + LSCIQuotaFeature = features.LocalStorageCapacityIsolationFSQuotaMonitoring +) + +func runOneQuotaTest(f *framework.Framework, quotasRequested bool) { + evictionTestTimeout := 10 * time.Minute + sizeLimit := resource.MustParse("100Mi") + useOverLimit := 101 /* Mb */ + useUnderLimit := 99 /* Mb */ + // TODO: remove hardcoded kubelet volume directory path + // framework.TestContext.KubeVolumeDir is currently not populated for node e2e + // As for why we do this: see comment below at isXfs. + if isXfs("/var/lib/kubelet") { + useUnderLimit = 50 /* Mb */ + } + priority := 0 + if quotasRequested { + priority = 1 + } + Context(fmt.Sprintf(testContextFmt, fmt.Sprintf("use quotas for LSCI monitoring (quotas enabled: %v)", quotasRequested)), func() { + tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { + defer withFeatureGate(LSCIQuotaFeature, quotasRequested)() + // TODO: remove hardcoded kubelet volume directory path + // framework.TestContext.KubeVolumeDir is currently not populated for node e2e + if quotasRequested && !supportsQuotas("/var/lib/kubelet") { + // No point in running this as a positive test if quotas are not + // enabled on the underlying filesystem. + framework.Skipf("Cannot run LocalStorageCapacityIsolationQuotaMonitoring on filesystem without project quota enabled") + } + // setting a threshold to 0% disables; non-empty map overrides default value (necessary due to omitempty) + initialConfig.EvictionHard = map[string]string{"memory.available": "0%"} + initialConfig.FeatureGates[string(LSCIQuotaFeature)] = quotasRequested + }) + runEvictionTest(f, evictionTestTimeout, noPressure, noStarvedResource, logDiskMetrics, []podEvictSpec{ + { + evictionPriority: priority, // This pod should be evicted because of emptyDir violation only if quotas are enabled + pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-over-sizelimit-quotas-%v", quotasRequested), useOverLimit, &v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, + }, v1.ResourceRequirements{}), + }, + { + evictionPriority: 0, // This pod should not be evicted because it uses less than its limit (test for quotas) + pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-under-sizelimit-quotas-%v", quotasRequested), useUnderLimit, &v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, + }, v1.ResourceRequirements{}), + }, + }) + }) +} + +// LocalStorageCapacityIsolationQuotaMonitoring tests that quotas are +// used for monitoring rather than du. The mechanism is to create a +// pod that creates a file, deletes it, and writes data to it. If +// quotas are used to monitor, it will detect this deleted-but-in-use +// file; if du is used to monitor, it will not detect this. +var _ = framework.KubeDescribe("LocalStorageCapacityIsolationQuotaMonitoring [Slow] [Serial] [Disruptive] [Feature:LocalStorageCapacityIsolationQuota][NodeFeature:LSCIQuotaMonitoring]", func() { + f := framework.NewDefaultFramework("localstorage-quota-monitoring-test") + runOneQuotaTest(f, true) + runOneQuotaTest(f, false) +}) + +const ( + writeConcealedPodCommand = ` +my $file = "%s.bin"; +open OUT, ">$file" || die "Cannot open $file: $!\n"; +unlink "$file" || die "Cannot unlink $file: $!\n"; +my $a = "a"; +foreach (1..20) { $a = "$a$a"; } +foreach (1..%d) { syswrite(OUT, $a); } +sleep 999999;` +) + +// This is needed for testing eviction of pods using disk space in concealed files; the shell has no convenient +// way of performing I/O to a concealed file, and the busybox image doesn't contain Perl. +func diskConcealingPod(name string, diskConsumedMB int, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod { + path := "" + volumeMounts := []v1.VolumeMount{} + volumes := []v1.Volume{} + if volumeSource != nil { + path = volumeMountPath + volumeMounts = []v1.VolumeMount{{MountPath: volumeMountPath, Name: volumeName}} + volumes = []v1.Volume{{Name: volumeName, VolumeSource: *volumeSource}} + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-pod", name)}, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Image: imageutils.GetE2EImage(imageutils.Perl), + Name: fmt.Sprintf("%s-container", name), + Command: []string{ + "perl", + "-e", + fmt.Sprintf(writeConcealedPodCommand, filepath.Join(path, "file"), diskConsumedMB), + }, + Resources: resources, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + } +} + +// Don't bother returning an error; if something goes wrong, +// simply treat it as "no". +func supportsQuotas(dir string) bool { + supportsQuota, err := quota.SupportsQuotas(mount.New(""), dir) + return supportsQuota && err == nil +} diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index cb78d4289a4..9db6272f016 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -34,7 +34,9 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/component-base/featuregate" internalapi "k8s.io/cri-api/pkg/apis" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" @@ -62,6 +64,7 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H var startServices = flag.Bool("start-services", true, "If true, start local node services") var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests") var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox) +var perlImage = imageutils.GetE2EImage(imageutils.Perl) const ( // Kubelet internal cgroup name for node allocatable cgroup. @@ -440,3 +443,15 @@ func reduceAllocatableMemoryUsage() { _, err := exec.Command("sudo", "sh", "-c", cmd).CombinedOutput() framework.ExpectNoError(err) } + +// Equivalent of featuregatetesting.SetFeatureGateDuringTest +// which can't be used here because we're not in a Testing context. +// This must be in a non-"_test" file to pass +// make verify WHAT=test-featuregates +func withFeatureGate(feature featuregate.Feature, desired bool) func() { + current := utilfeature.DefaultFeatureGate.Enabled(feature) + utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), desired)) + return func() { + utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), current)) + } +} diff --git a/test/e2e_node/util_xfs_linux.go b/test/e2e_node/util_xfs_linux.go new file mode 100644 index 00000000000..5f758134a0b --- /dev/null +++ b/test/e2e_node/util_xfs_linux.go @@ -0,0 +1,74 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "path/filepath" + "syscall" + + "k8s.io/kubernetes/pkg/util/mount" +) + +func detectMountpoint(m mount.Interface, path string) string { + path, err := filepath.Abs(path) + if err == nil { + path, err = filepath.EvalSymlinks(path) + } + if err != nil { + return "" + } + for path != "" && path != "/" { + isNotMount, err := m.IsLikelyNotMountPoint(path) + if err != nil { + return "" + } + if !isNotMount { + return path + } + path = filepath.Dir(path) + } + return "/" +} + +const ( + xfsMagic = 0x58465342 +) + +// XFS over-allocates and then eventually removes that excess allocation. +// That can lead to a file growing beyond its eventual size, causing +// an unnecessary eviction: +// +// % ls -ls +// total 32704 +// 32704 -rw-r--r-- 1 rkrawitz rkrawitz 20971520 Jan 15 13:16 foo.bin +// +// This issue can be hit regardless of the means used to count storage. +// It is not present in ext4fs. +func isXfs(dir string) bool { + mountpoint := detectMountpoint(mount.New(""), dir) + if mountpoint == "" { + return false + } + var buf syscall.Statfs_t + err := syscall.Statfs(mountpoint, &buf) + if err != nil { + return false + } + return buf.Type == xfsMagic +} diff --git a/test/e2e_node/util_xfs_unsupported.go b/test/e2e_node/util_xfs_unsupported.go new file mode 100644 index 00000000000..01fdb802b30 --- /dev/null +++ b/test/e2e_node/util_xfs_unsupported.go @@ -0,0 +1,23 @@ +// +build !linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +func isXfs(dir string) bool { + return false +} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index dc6b2e1ad26..6fb3db8ea85 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -171,6 +171,8 @@ const ( // Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go // Pause image Pause + // Perl image + Perl // Porter image Porter // PortForwardTester image @@ -236,6 +238,7 @@ func initImageConfigs() map[int]Config { configs[NoSnatTestProxy] = Config{e2eRegistry, "no-snat-test-proxy", "1.0"} // Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go configs[Pause] = Config{gcRegistry, "pause", "3.1"} + configs[Perl] = Config{dockerLibraryRegistry, "perl", "5.26"} configs[Porter] = Config{e2eRegistry, "porter", "1.0"} configs[PortForwardTester] = Config{e2eRegistry, "port-forward-tester", "1.0"} configs[Redis] = Config{e2eRegistry, "redis", "1.0"}