From bb7cff161c582a30f2209849d51f8ec00363ea6b Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Mon, 11 Aug 2025 10:22:58 +0200 Subject: [PATCH] e2e: node: split utilities to learn machine properties A lot ofe2e_node tests need to re-learn machine HW properties to check the correctness of the behavior. Over time, we start using these utilities among different test groups (e.g. memory manager tests use cpu manager tests utilites). So let's de-entangle this state by moving the shared utilities in a separate util file. Trivial code movement, no intended behavioral changes. Signed-off-by: Francesco Romani --- test/e2e_node/cpu_manager_test.go | 89 ------------- test/e2e_node/util_machineinfo_linux.go | 124 ++++++++++++++++++ ...ted.go => util_machineinfo_unsupported.go} | 37 +++++- test/e2e_node/utils_linux.go | 6 - 4 files changed, 160 insertions(+), 96 deletions(-) create mode 100644 test/e2e_node/util_machineinfo_linux.go rename test/e2e_node/{utils_unsupported.go => util_machineinfo_unsupported.go} (57%) diff --git a/test/e2e_node/cpu_manager_test.go b/test/e2e_node/cpu_manager_test.go index d27c76c841c..35aa6504acb 100644 --- a/test/e2e_node/cpu_manager_test.go +++ b/test/e2e_node/cpu_manager_test.go @@ -18,14 +18,8 @@ package e2enode import ( "context" - "errors" "fmt" - "io/fs" - "os" - "os/exec" - "path/filepath" "regexp" - "strconv" "strings" "time" @@ -234,62 +228,6 @@ func waitForContainerRemoval(ctx context.Context, containerName, podName, podNS }, 2*time.Minute, 1*time.Second).Should(gomega.BeTrueBecause("Containers were expected to be removed")) } -func isHTEnabled() bool { - outData, err := exec.Command("/bin/sh", "-c", "lscpu | grep \"Thread(s) per core:\" | cut -d \":\" -f 2").Output() - framework.ExpectNoError(err) - - threadsPerCore, err := strconv.Atoi(strings.TrimSpace(string(outData))) - framework.ExpectNoError(err) - - return threadsPerCore > 1 -} - -func isMultiNUMA() bool { - outData, err := exec.Command("/bin/sh", "-c", "lscpu | grep \"NUMA node(s):\" | cut -d \":\" -f 2").Output() - framework.ExpectNoError(err) - - numaNodes, err := strconv.Atoi(strings.TrimSpace(string(outData))) - framework.ExpectNoError(err) - - return numaNodes > 1 -} - -func getSMTLevel() int { - cpuID := 0 // this is just the most likely cpu to be present in a random system. No special meaning besides this. - out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/thread_siblings_list | tr -d \"\n\r\"", cpuID)).Output() - framework.ExpectNoError(err) - // how many thread sibling you have = SMT level - // example: 2-way SMT means 2 threads sibling for each thread - cpus, err := cpuset.Parse(strings.TrimSpace(string(out))) - framework.ExpectNoError(err) - return cpus.Size() -} - -func getUncoreCPUGroupSize() int { - cpuID := 0 // this is just the most likely cpu to be present in a random system. No special meaning besides this. - out, err := os.ReadFile(fmt.Sprintf("/sys/devices/system/cpu/cpu%d/cache/index3/shared_cpu_list", cpuID)) - if errors.Is(err, fs.ErrNotExist) { - return 0 // no Uncore/LLC cache detected, nothing to do - } - framework.ExpectNoError(err) - // how many cores share a same Uncore/LLC block? - cpus, err := cpuset.Parse(strings.TrimSpace(string(out))) - framework.ExpectNoError(err) - return cpus.Size() -} - -func getCPUSiblingList(cpuRes int64) string { - out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/thread_siblings_list | tr -d \"\n\r\"", cpuRes)).Output() - framework.ExpectNoError(err) - return string(out) -} - -func getCoreSiblingList(cpuRes int64) string { - out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/core_siblings_list | tr -d \"\n\r\"", cpuRes)).Output() - framework.ExpectNoError(err) - return string(out) -} - type cpuManagerKubeletArguments struct { policyName string enableCPUManagerOptions bool @@ -1400,33 +1338,6 @@ func isSMTAlignmentError(pod *v1.Pod) bool { return re.MatchString(pod.Status.Reason) } -// getNumaNodeCPUs retrieves CPUs for each NUMA node. -func getNumaNodeCPUs() (map[int]cpuset.CPUSet, error) { - numaNodes := make(map[int]cpuset.CPUSet) - nodePaths, err := filepath.Glob("/sys/devices/system/node/node*/cpulist") - if err != nil { - return nil, err - } - - for _, nodePath := range nodePaths { - data, err := os.ReadFile(nodePath) - framework.ExpectNoError(err, "Error obtaning CPU information from the node") - cpuSet := strings.TrimSpace(string(data)) - cpus, err := cpuset.Parse(cpuSet) - framework.ExpectNoError(err, "Error parsing CPUset") - - // Extract node ID from path (e.g., "node0" -> 0) - base := filepath.Base(filepath.Dir(nodePath)) - nodeID, err := strconv.Atoi(strings.TrimPrefix(base, "node")) - if err != nil { - continue - } - numaNodes[nodeID] = cpus - } - - return numaNodes, nil -} - func getContainerAllowedCPUsFromLogs(podName, cntName, logs string) cpuset.CPUSet { framework.Logf("got pod logs: <%v>", logs) cpus, err := cpuset.Parse(strings.TrimSpace(logs)) diff --git a/test/e2e_node/util_machineinfo_linux.go b/test/e2e_node/util_machineinfo_linux.go new file mode 100644 index 00000000000..d89e8e28c22 --- /dev/null +++ b/test/e2e_node/util_machineinfo_linux.go @@ -0,0 +1,124 @@ +//go:build linux +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + libcontainercgroups "github.com/opencontainers/cgroups" + "k8s.io/utils/cpuset" + + "k8s.io/kubernetes/test/e2e/framework" +) + +// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode. +func IsCgroup2UnifiedMode() bool { + return libcontainercgroups.IsCgroup2UnifiedMode() +} + +func isHTEnabled() bool { + outData, err := exec.Command("/bin/sh", "-c", "lscpu | grep \"Thread(s) per core:\" | cut -d \":\" -f 2").Output() + framework.ExpectNoError(err) + + threadsPerCore, err := strconv.Atoi(strings.TrimSpace(string(outData))) + framework.ExpectNoError(err) + + return threadsPerCore > 1 +} + +func isMultiNUMA() bool { + outData, err := exec.Command("/bin/sh", "-c", "lscpu | grep \"NUMA node(s):\" | cut -d \":\" -f 2").Output() + framework.ExpectNoError(err) + + numaNodes, err := strconv.Atoi(strings.TrimSpace(string(outData))) + framework.ExpectNoError(err) + + return numaNodes > 1 +} + +func getSMTLevel() int { + cpuID := 0 // this is just the most likely cpu to be present in a random system. No special meaning besides this. + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/thread_siblings_list | tr -d \"\n\r\"", cpuID)).Output() + framework.ExpectNoError(err) + // how many thread sibling you have = SMT level + // example: 2-way SMT means 2 threads sibling for each thread + cpus, err := cpuset.Parse(strings.TrimSpace(string(out))) + framework.ExpectNoError(err) + return cpus.Size() +} + +func getUncoreCPUGroupSize() int { + cpuID := 0 // this is just the most likely cpu to be present in a random system. No special meaning besides this. + out, err := os.ReadFile(fmt.Sprintf("/sys/devices/system/cpu/cpu%d/cache/index3/shared_cpu_list", cpuID)) + if errors.Is(err, fs.ErrNotExist) { + return 0 // no Uncore/LLC cache detected, nothing to do + } + framework.ExpectNoError(err) + // how many cores share a same Uncore/LLC block? + cpus, err := cpuset.Parse(strings.TrimSpace(string(out))) + framework.ExpectNoError(err) + return cpus.Size() +} + +func getCPUSiblingList(cpuRes int64) string { + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/thread_siblings_list | tr -d \"\n\r\"", cpuRes)).Output() + framework.ExpectNoError(err) + return string(out) +} + +func getCoreSiblingList(cpuRes int64) string { + out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("cat /sys/devices/system/cpu/cpu%d/topology/core_siblings_list | tr -d \"\n\r\"", cpuRes)).Output() + framework.ExpectNoError(err) + return string(out) +} + +// getNumaNodeCPUs retrieves CPUs for each NUMA node. +func getNumaNodeCPUs() (map[int]cpuset.CPUSet, error) { + numaNodes := make(map[int]cpuset.CPUSet) + nodePaths, err := filepath.Glob("/sys/devices/system/node/node*/cpulist") + if err != nil { + return nil, err + } + + for _, nodePath := range nodePaths { + data, err := os.ReadFile(nodePath) + framework.ExpectNoError(err, "Error obtaning CPU information from the node") + cpuSet := strings.TrimSpace(string(data)) + cpus, err := cpuset.Parse(cpuSet) + framework.ExpectNoError(err, "Error parsing CPUset") + + // Extract node ID from path (e.g., "node0" -> 0) + base := filepath.Base(filepath.Dir(nodePath)) + nodeID, err := strconv.Atoi(strings.TrimPrefix(base, "node")) + if err != nil { + continue + } + numaNodes[nodeID] = cpus + } + + return numaNodes, nil +} diff --git a/test/e2e_node/utils_unsupported.go b/test/e2e_node/util_machineinfo_unsupported.go similarity index 57% rename from test/e2e_node/utils_unsupported.go rename to test/e2e_node/util_machineinfo_unsupported.go index 758af2798ab..a5d6687706e 100644 --- a/test/e2e_node/utils_unsupported.go +++ b/test/e2e_node/util_machineinfo_unsupported.go @@ -2,7 +2,7 @@ // +build !linux /* -Copyright 2020 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,42 @@ limitations under the License. package e2enode +import ( + "errors" + + "k8s.io/utils/cpuset" +) + // IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode. func IsCgroup2UnifiedMode() bool { return false } + +func isHTEnabled() bool { + return false +} + +func isMultiNUMA() bool { + return false +} + +func getSMTLevel() int { + return 1 +} + +func getUncoreCPUGroupSize() int { + return 1 +} + +func getCPUSiblingList(cpuRes int64) string { + return "" +} + +func getCoreSiblingList(cpuRes int64) string { + return "" +} + +// getNumaNodeCPUs retrieves CPUs for each NUMA node. +func getNumaNodeCPUs() (map[int]cpuset.CPUSet, error) { + return nil, errors.New("not implemented") +} diff --git a/test/e2e_node/utils_linux.go b/test/e2e_node/utils_linux.go index b4c86c0c961..a33b6a06c2c 100644 --- a/test/e2e_node/utils_linux.go +++ b/test/e2e_node/utils_linux.go @@ -22,15 +22,9 @@ package e2enode import ( "fmt" - libcontainercgroups "github.com/opencontainers/cgroups" "k8s.io/kubernetes/test/e2e_node/criproxy" ) -// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode. -func IsCgroup2UnifiedMode() bool { - return libcontainercgroups.IsCgroup2UnifiedMode() -} - // addCRIProxyInjector registers an injector function for the CRIProxy. func addCRIProxyInjector(proxy *criproxy.RemoteRuntime, injector func(apiName string) error) error { if proxy == nil {