github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-18 04:54:54 +00:00
Code Issues Projects Releases Wiki Activity

e2e: node: rewrite: multi-cpus single-container pods

Browse Source 
We have tests which cover the case on which a pod
with a single container require multiple CPUs;
rewrite them preserving the testcases and actually
adding coverage.

Add and use stricter checks along the way.

Signed-off-by: Francesco Romani <fromani@redhat.com>
This commit is contained in:
Francesco Romani
2025-05-06 15:09:25 +02:00
parent e4726719a7
commit 2419d9ccc5
1 changed files with 148 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
test/e2e_node/cpumanager_test.go
View File

@@ -212,7 +212,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
gomega.Expect(pod).To(HaveContainerCPUsEqualTo("non-gu-container", onlineCPUs))
})
ginkgo.It("should let the container access all the online non-exclusively-allocated CPUs when using a reserved CPUs set", Label("guaranteed", "exclusive-cpus"), func(ctx context.Context) {
ginkgo.It("should let the container access all the online non-exclusively-allocated CPUs when using a reserved CPUs set", ginkgo.Label("guaranteed", "exclusive-cpus"), func(ctx context.Context) {
cpuCount := 1
reservedCPUs := cpuset.New(0)
// TODO: we assume the first available CPUID is 0, which is pretty fair, but we should probably
@@ -249,7 +249,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
// we cannot nor we should predict which CPUs the container gets
gomega.Expect(podGu).To(HaveContainerCPUsCount("gu-container", cpuCount))
gomega.Expect(podGu).To(HaveContainerCPUsOverlapWith("gu-container", onlineCPUs))
gomega.Expect(podGu).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
gomega.Expect(podGu).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
exclusiveCPUs, err := getContainerAllowedCPUs(podGu, "gu-container")
@@ -264,7 +264,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
cpuCount := 1
reservedCPUs := cpuset.New(0)
// TODO: we assume the first available CPUID is 0, which is pretty fair, but we should probably
// check what we do have in the node.
// check what we do have in the node. -- note this snippet is repeated multiple time over the tests
_ = updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
@@ -286,9 +286,77 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
// we cannot nor we should predict which CPUs the container gets
gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
gomega.Expect(pod).To(HaveContainerCPUsOverlapWith("gu-container", onlineCPUs))
gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
})
// we don't use a separate group (gingo.When) with BeforeEach to factor out the tests because each
// test need to check for the amount of CPUs it needs.
ginkgo.It("should allocate exclusively a even number of CPUs to a 1-container pod", func(ctx context.Context) {
cpuCount := 2
reservedCPUs := cpuset.New(0)
skipIfNotEnoughAllocatableCPUs(ctx, f, cpuCount+reservedCPUs.Size(), onlineCPUs)
_ = updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
}))
pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
{
ctnName: "gu-container",
cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
cpuLimit: fmt.Sprintf("%dm", 1000*cpuCount),
},
})
ginkgo.By("creating the test pod")
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
ginkgo.By("checking if the expected cpuset was assigned")
// we cannot nor we should predict which CPUs the container gets
gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
// TODO: this is probably too strict but it is the closest of the old test did
gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings("gu-container"))
})
ginkgo.It("should allocate exclusively a odd number of CPUs to a 1-container pod", func(ctx context.Context) {
cpuCount := 3
reservedCPUs := cpuset.New(0)
skipIfNotEnoughAllocatableCPUs(ctx, f, cpuCount+reservedCPUs.Size(), onlineCPUs)
_ = updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check
}))
pod := makeCPUManagerPod("gu-pod", []ctnAttribute{
{
ctnName: "gu-container",
cpuRequest: fmt.Sprintf("%dm", 1000*cpuCount),
cpuLimit: fmt.Sprintf("%dm", 1000*cpuCount),
},
})
ginkgo.By("creating the test pod")
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podMap[string(pod.UID)] = pod
ginkgo.By("checking if the expected cpuset was assigned")
// we cannot nor we should predict which CPUs the container gets
gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", cpuCount))
gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs))
gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
// TODO: this is probably too strict but it is the closest of the old test did
toleration := 1
gomega.Expect(pod).To(HaveContainerCPUsQuasiThreadSiblings("gu-container", toleration))
})
})
ginkgo.When("running with strict CPU reservation", ginkgo.Label("strict-cpu-reservation"), func() {
@@ -395,7 +463,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
// any full CPU is fine - we cannot nor we should predict which one, though
gomega.Expect(podGu).To(HaveContainerCPUsCount("gu-container", cpuCount))
gomega.Expect(podGu).To(HaveContainerCPUsOverlapWith("gu-container", usableCPUs))
gomega.Expect(podGu).To(HaveContainerCPUsASubsetOf("gu-container", usableCPUs))
gomega.Expect(podGu).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs))
exclusiveCPUs, err := getContainerAllowedCPUs(podGu, "gu-container")
@@ -494,11 +562,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))
gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
cpus, err := getContainerAllowedCPUs(pod, cnt.Name)
framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)
siblingsCPUs := makeThreadSiblingCPUSet(cpus)
gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))
gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
}
})
})
@@ -607,7 +671,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
usableCPUs := onlineCPUs.Difference(reservedCPUs)
gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-x", cpuCount))
gomega.Expect(pod).To(HaveContainerCPUsOverlapWith("gu-container-x", usableCPUs))
gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-x", usableCPUs))
gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-x", reservedCPUs))
ginkgo.By("validating each container in the testing pod")
@@ -615,11 +679,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))
gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
cpus, err := getContainerAllowedCPUs(pod, cnt.Name)
framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)
siblingsCPUs := makeThreadSiblingCPUSet(cpus)
gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))
gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
}
})
})
@@ -676,12 +736,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, framework.WithSerial(), featu
ginkgo.By(fmt.Sprintf("validating the container %s on pod %s", cnt.Name, pod.Name))
gomega.Expect(pod).To(HaveContainerCPUsAlignedTo(cnt.Name, smtLevel))
cpus, err := getContainerAllowedCPUs(pod, cnt.Name)
framework.ExpectNoError(err, "cannot get cpus allocated to pod %s/%s cnt %s", pod.Namespace, pod.Name, cnt.Name)
gomega.Expect(cpus).To(BePackedCPUs())
siblingsCPUs := makeThreadSiblingCPUSet(cpus)
gomega.Expect(pod).To(HaveContainerCPUsEqualTo(cnt.Name, siblingsCPUs))
gomega.Expect(pod).To(HaveContainerCPUsThreadSiblings(cnt.Name))
}
})
@@ -904,13 +959,14 @@ func HaveStatusReasonMatchingRegex(expr string) types.GomegaMatcher {
}
type msgData struct {
Name string
CurrentCPUs string
ExpectedCPUs string
Count int
Aligned int
CurrentQuota string
ExpectedQuota string
Name string
CurrentCPUs string
ExpectedCPUs string
MismatchedCPUs string
Count int
Aligned int
CurrentQuota string
ExpectedQuota string
}
func HaveContainerCPUsCount(ctnName string, val int) types.GomegaMatcher {
@@ -962,6 +1018,22 @@ func HaveContainerCPUsOverlapWith(ctnName string, ref cpuset.CPUSet) types.Gomeg
}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> overlapping with expected CPUs <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md)
}
func HaveContainerCPUsASubsetOf(ctnName string, ref cpuset.CPUSet) types.GomegaMatcher {
md := &msgData{
Name: ctnName,
ExpectedCPUs: ref.String(),
}
return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
cpus, err := getContainerAllowedCPUs(actual, ctnName)
md.CurrentCPUs = cpus.String()
if err != nil {
framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
return false, err
}
return cpus.IsSubsetOf(ref), nil
}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not a subset of expected CPUs <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md)
}
func HaveContainerCPUsEqualTo(ctnName string, expectedCPUs cpuset.CPUSet) types.GomegaMatcher {
md := &msgData{
Name: ctnName,
@@ -1018,8 +1090,55 @@ func HaveContainerQuota(ctnName, expectedQuota string) types.GomegaMatcher {
}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has quota <{{.Data.CurrentQuota}}> not matching expected value <{{.Data.ExpectedQuota}}> for container {{.Data.Name}}", md)
}
func HaveContainerCPUsThreadSiblings(ctnName string) types.GomegaMatcher {
md := &msgData{
Name: ctnName,
}
return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
cpus, err := getContainerAllowedCPUs(actual, ctnName)
md.CurrentCPUs = cpus.String()
if err != nil {
framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
return false, err
}
expectedCPUs := makeThreadSiblingCPUSet(cpus)
md.ExpectedCPUs = expectedCPUs.String()
return cpus.Equals(expectedCPUs), nil
}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not all thread sibling pairs (would be <{{.Data.ExpectedCPUs}}>) for container {{.Data.Name}}", md)
}
func HaveContainerCPUsQuasiThreadSiblings(ctnName string, toleration int) types.GomegaMatcher {
md := &msgData{
Name: ctnName,
Count: toleration,
}
return gcustom.MakeMatcher(func(actual *v1.Pod) (bool, error) {
cpus, err := getContainerAllowedCPUs(actual, ctnName)
md.CurrentCPUs = cpus.String()
if err != nil {
framework.Logf("getContainerAllowedCPUs(%s) failed: %v", ctnName, err)
return false, err
}
// this is by construction >= cpus (extreme case: cpus is made by all non-thread-siblings)
expectedCPUs := makeThreadSiblingCPUSet(cpus)
md.ExpectedCPUs = expectedCPUs.String()
mismatchedCPUs := expectedCPUs.Difference(cpus)
md.MismatchedCPUs = mismatchedCPUs.String()
return mismatchedCPUs.Size() <= toleration, nil
}).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> not all thread sibling pairs (would be <{{.Data.ExpectedCPUs}}> mismatched <{{.Data.MismatchedCPUs}}> toleration <{{.Data.Count}}>) for container {{.Data.Name}}", md)
}
// Other helpers
func skipIfNotEnoughAllocatableCPUs(ctx context.Context, f *framework.Framework, cpuReq int, onlineCPUs cpuset.CPUSet) {
localNode := getLocalNode(ctx, f)
cpuAllocQty := localNode.Status.Allocatable[v1.ResourceCPU]
cpuAlloc := cpuAllocQty.Value()
if cpuAlloc < int64(cpuReq) {
ginkgo.Skip(fmt.Sprintf("Skipping CPU Manager tests since the CPU allocatable %d < CPU request %d (online %d)", cpuAlloc, cpuReq, onlineCPUs.Size()), 1)
}
}
func getContainerAllowedCPUs(pod *v1.Pod, ctnName string) (cpuset.CPUSet, error) {
cgPath, err := makeCgroupPathForContainer(pod, ctnName)
if err != nil {