mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
e2e: node: cpumgr: remove old smt alignment tests
superseded by code in cpumanager_test.go, which will be moved in cpu_manager_test.go at the end of this series. Split to make the review easier. Signed-off-by: Francesco Romani <fromani@redhat.com>
This commit is contained in:
Francesco Romani
@@ -19,12 +19,8 @@ package e2enode
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
@@ -190,81 +186,6 @@ func runCPUManagerTests(f *framework.Framework) {
|
||||
runNonGuPodTest(ctx, f, cpuCap, reservedSystemCPUs)
|
||||
})
|
||||
|
||||
ginkgo.It("should assign CPUs as expected with enhanced policy based on strict SMT alignment", func(ctx context.Context) {
|
||||
fullCPUsOnlyOpt := fmt.Sprintf("option=%s", cpumanager.FullPCPUsOnlyOption)
|
||||
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
|
||||
smtLevel := getSMTLevel()
|
||||
|
||||
// strict SMT alignment is trivially verified and granted on non-SMT systems
|
||||
if smtLevel < minSMTLevel {
|
||||
e2eskipper.Skipf("Skipping CPU Manager %s tests since SMT disabled", fullCPUsOnlyOpt)
|
||||
}
|
||||
|
||||
// our tests want to allocate a full core, so we need at least 2*2=4 virtual cpus
|
||||
minCPUCount := int64(smtLevel * minCPUCapacity)
|
||||
if cpuAlloc < minCPUCount {
|
||||
e2eskipper.Skipf("Skipping CPU Manager %s tests since the CPU capacity < %d", fullCPUsOnlyOpt, minCPUCount)
|
||||
}
|
||||
|
||||
framework.Logf("SMT level %d", smtLevel)
|
||||
|
||||
// TODO: we assume the first available CPUID is 0, which is pretty fair, but we should probably
|
||||
// check what we do have in the node.
|
||||
cpuPolicyOptions := map[string]string{
|
||||
cpumanager.FullPCPUsOnlyOption: "true",
|
||||
}
|
||||
newCfg := configureCPUManagerInKubelet(oldCfg,
|
||||
&cpuManagerKubeletArguments{
|
||||
policyName: string(cpumanager.PolicyStatic),
|
||||
reservedSystemCPUs: cpuset.New(0),
|
||||
enableCPUManagerOptions: true,
|
||||
options: cpuPolicyOptions,
|
||||
},
|
||||
)
|
||||
updateKubeletConfig(ctx, f, newCfg, true)
|
||||
|
||||
// the order between negative and positive doesn't really matter
|
||||
runSMTAlignmentNegativeTests(ctx, f)
|
||||
runSMTAlignmentPositiveTests(ctx, f, smtLevel, cpuset.New())
|
||||
})
|
||||
|
||||
ginkgo.It("should assign CPUs as expected based on strict SMT alignment, reservedSystemCPUs should be excluded (both strict-cpu-reservation and full-pcpus-only options enabled)", func(ctx context.Context) {
|
||||
fullCPUsOnlyOpt := fmt.Sprintf("option=%s", cpumanager.FullPCPUsOnlyOption)
|
||||
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
|
||||
smtLevel := getSMTLevel()
|
||||
|
||||
// strict SMT alignment is trivially verified and granted on non-SMT systems
|
||||
if smtLevel < 2 {
|
||||
e2eskipper.Skipf("Skipping CPU Manager %s tests since SMT disabled", fullCPUsOnlyOpt)
|
||||
}
|
||||
|
||||
// our tests want to allocate a full core, so we need at last smtLevel*2 virtual cpus
|
||||
if cpuAlloc < int64(smtLevel*2) {
|
||||
e2eskipper.Skipf("Skipping CPU Manager %s tests since the CPU capacity < %d", fullCPUsOnlyOpt, smtLevel*2)
|
||||
}
|
||||
|
||||
framework.Logf("SMT level %d", smtLevel)
|
||||
|
||||
reservedSystemCPUs := cpuset.New(0)
|
||||
cpuPolicyOptions := map[string]string{
|
||||
cpumanager.FullPCPUsOnlyOption: "true",
|
||||
cpumanager.StrictCPUReservationOption: "true",
|
||||
}
|
||||
newCfg := configureCPUManagerInKubelet(oldCfg,
|
||||
&cpuManagerKubeletArguments{
|
||||
policyName: string(cpumanager.PolicyStatic),
|
||||
reservedSystemCPUs: reservedSystemCPUs,
|
||||
enableCPUManagerOptions: true,
|
||||
options: cpuPolicyOptions,
|
||||
},
|
||||
)
|
||||
updateKubeletConfig(ctx, f, newCfg, true)
|
||||
|
||||
// the order between negative and positive doesn't really matter
|
||||
runSMTAlignmentNegativeTests(ctx, f)
|
||||
runSMTAlignmentPositiveTests(ctx, f, smtLevel, reservedSystemCPUs)
|
||||
})
|
||||
|
||||
f.It("should not reuse CPUs of restartable init containers", feature.SidecarContainers, func(ctx context.Context) {
|
||||
cpuCap, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
|
||||
|
||||
@@ -473,106 +394,6 @@ func runCPUManagerTests(f *framework.Framework) {
|
||||
})
|
||||
}
|
||||
|
||||
func runSMTAlignmentNegativeTests(ctx context.Context, f *framework.Framework) {
|
||||
// negative test: try to run a container whose requests aren't a multiple of SMT level, expect a rejection
|
||||
ctnAttrs := []ctnAttribute{
|
||||
{
|
||||
ctnName: "gu-container-neg",
|
||||
cpuRequest: "1000m",
|
||||
cpuLimit: "1000m",
|
||||
},
|
||||
}
|
||||
pod := makeCPUManagerPod("gu-pod", ctnAttrs)
|
||||
// CreateSync would wait for pod to become Ready - which will never happen if production code works as intended!
|
||||
pod = e2epod.NewPodClient(f).Create(ctx, pod)
|
||||
|
||||
err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Failed", 30*time.Second, func(pod *v1.Pod) (bool, error) {
|
||||
if pod.Status.Phase != v1.PodPending {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
framework.ExpectNoError(err)
|
||||
pod, err = e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
if pod.Status.Phase != v1.PodFailed {
|
||||
framework.Failf("pod %s not failed: %v", pod.Name, pod.Status)
|
||||
}
|
||||
if !isSMTAlignmentError(pod) {
|
||||
framework.Failf("pod %s failed for wrong reason: %q", pod.Name, pod.Status.Reason)
|
||||
}
|
||||
|
||||
deletePodSyncByName(ctx, f, pod.Name)
|
||||
// we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state.
|
||||
// this is in turn needed because we will have an unavoidable (in the current framework) race with th
|
||||
// reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire
|
||||
waitForAllContainerRemoval(ctx, pod.Name, pod.Namespace)
|
||||
}
|
||||
|
||||
func runSMTAlignmentPositiveTests(ctx context.Context, f *framework.Framework, smtLevel int, strictReservedCPUs cpuset.CPUSet) {
|
||||
// positive test: try to run a container whose requests are a multiple of SMT level, check allocated cores
|
||||
// 1. are core siblings
|
||||
// 2. take a full core
|
||||
// WARNING: this assumes 2-way SMT systems - we don't know how to access other SMT levels.
|
||||
// this means on more-than-2-way SMT systems this test will prove nothing
|
||||
ctnAttrs := []ctnAttribute{
|
||||
{
|
||||
ctnName: "gu-container-pos",
|
||||
cpuRequest: "2000m",
|
||||
cpuLimit: "2000m",
|
||||
},
|
||||
}
|
||||
pod := makeCPUManagerPod("gu-pod", ctnAttrs)
|
||||
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
|
||||
|
||||
for _, cnt := range pod.Spec.Containers {
|
||||
ginkgo.By(fmt.Sprintf("validating the container %s on Gu pod %s", cnt.Name, pod.Name))
|
||||
|
||||
logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, pod.Name, cnt.Name)
|
||||
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]", cnt.Name, pod.Name)
|
||||
|
||||
cpus := getContainerAllowedCPUsFromLogs(pod.Name, cnt.Name, logs)
|
||||
|
||||
gomega.Expect(cpus.Intersection(strictReservedCPUs).IsEmpty()).To(gomega.BeTrueBecause("cpuset %q should not contain strict reserved cpus %q", cpus.String(), strictReservedCPUs.String()))
|
||||
validateSMTAlignment(cpus, smtLevel, pod, &cnt)
|
||||
}
|
||||
|
||||
deletePodSyncByName(ctx, f, pod.Name)
|
||||
// we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state.
|
||||
// this is in turn needed because we will have an unavoidable (in the current framework) race with th
|
||||
// reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire
|
||||
waitForAllContainerRemoval(ctx, pod.Name, pod.Namespace)
|
||||
}
|
||||
|
||||
func validateSMTAlignment(cpus cpuset.CPUSet, smtLevel int, pod *v1.Pod, cnt *v1.Container) {
|
||||
framework.Logf("validating cpus: %v", cpus)
|
||||
|
||||
if cpus.Size()%smtLevel != 0 {
|
||||
framework.Failf("pod %q cnt %q received non-smt-multiple cpuset %v (SMT level %d)", pod.Name, cnt.Name, cpus, smtLevel)
|
||||
}
|
||||
|
||||
// now check all the given cpus are thread siblings.
|
||||
// to do so the easiest way is to rebuild the expected set of siblings from all the cpus we got.
|
||||
// if the expected set matches the given set, the given set was good.
|
||||
siblingsCPUs := cpuset.New()
|
||||
for _, cpuID := range cpus.UnsortedList() {
|
||||
threadSiblings, err := cpuset.Parse(strings.TrimSpace(getCPUSiblingList(int64(cpuID))))
|
||||
framework.ExpectNoError(err, "parsing cpuset from logs for [%s] of pod [%s]", cnt.Name, pod.Name)
|
||||
siblingsCPUs = siblingsCPUs.Union(threadSiblings)
|
||||
}
|
||||
|
||||
framework.Logf("siblings cpus: %v", siblingsCPUs)
|
||||
if !siblingsCPUs.Equals(cpus) {
|
||||
framework.Failf("pod %q cnt %q received non-smt-aligned cpuset %v (expected %v)", pod.Name, cnt.Name, cpus, siblingsCPUs)
|
||||
}
|
||||
}
|
||||
|
||||
func isSMTAlignmentError(pod *v1.Pod) bool {
|
||||
re := regexp.MustCompile(`SMT.*Alignment.*Error`)
|
||||
return re.MatchString(pod.Status.Reason)
|
||||
}
|
||||
|
||||
// Serial because the test updates kubelet configuration.
|
||||
var _ = SIGDescribe("CPU Manager", framework.WithSerial(), feature.CPUManager, func() {
|
||||
f := framework.NewDefaultFramework("cpu-manager-test")
|
||||
|
||||
Reference in New Issue
Block a user