mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
e2e: topomgr: check pod resource alignment
This patch extends and completes the previously-added empty topology manager test for single-NUMA node policy by adding reporting in the test pod and checking the resource alignment. Signed-off-by: Francesco Romani <fromani@redhat.com>
This commit is contained in:
parent
cd7e3d626c
commit
fa26fb6817
@ -15,6 +15,7 @@ go_library(
|
|||||||
"framework.go",
|
"framework.go",
|
||||||
"image_list.go",
|
"image_list.go",
|
||||||
"node_problem_detector_linux.go",
|
"node_problem_detector_linux.go",
|
||||||
|
"numa_alignment.go",
|
||||||
"resource_collector.go",
|
"resource_collector.go",
|
||||||
"util.go",
|
"util.go",
|
||||||
"util_sriov.go",
|
"util_sriov.go",
|
||||||
@ -31,6 +32,7 @@ go_library(
|
|||||||
"//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
|
"//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
|
||||||
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
|
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
|
||||||
"//pkg/kubelet/cm:go_default_library",
|
"//pkg/kubelet/cm:go_default_library",
|
||||||
|
"//pkg/kubelet/cm/cpuset:go_default_library",
|
||||||
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
|
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
|
||||||
"//pkg/kubelet/metrics:go_default_library",
|
"//pkg/kubelet/metrics:go_default_library",
|
||||||
"//pkg/kubelet/remote:go_default_library",
|
"//pkg/kubelet/remote:go_default_library",
|
||||||
|
199
test/e2e_node/numa_alignment.go
Normal file
199
test/e2e_node/numa_alignment.go
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2020 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package e2enode
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
|
)
|
||||||
|
|
||||||
|
type numaPodResources struct {
|
||||||
|
CPUToNUMANode map[int]int
|
||||||
|
PCIDevsToNUMANode map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (R *numaPodResources) CheckAlignment() bool {
|
||||||
|
nodeNum := -1 // not set
|
||||||
|
for _, cpuNode := range R.CPUToNUMANode {
|
||||||
|
if nodeNum == -1 {
|
||||||
|
nodeNum = cpuNode
|
||||||
|
} else if nodeNum != cpuNode {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, devNode := range R.PCIDevsToNUMANode {
|
||||||
|
// TODO: explain -1
|
||||||
|
if devNode != -1 && nodeNum != devNode {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (R *numaPodResources) String() string {
|
||||||
|
var b strings.Builder
|
||||||
|
// To store the keys in slice in sorted order
|
||||||
|
var cpuKeys []int
|
||||||
|
for ck := range R.CPUToNUMANode {
|
||||||
|
cpuKeys = append(cpuKeys, ck)
|
||||||
|
}
|
||||||
|
sort.Ints(cpuKeys)
|
||||||
|
for _, k := range cpuKeys {
|
||||||
|
nodeNum := R.CPUToNUMANode[k]
|
||||||
|
b.WriteString(fmt.Sprintf("CPU cpu#%03d=%02d\n", k, nodeNum))
|
||||||
|
}
|
||||||
|
var pciKeys []string
|
||||||
|
for pk := range R.PCIDevsToNUMANode {
|
||||||
|
pciKeys = append(pciKeys, pk)
|
||||||
|
}
|
||||||
|
sort.Strings(pciKeys)
|
||||||
|
for _, k := range pciKeys {
|
||||||
|
nodeNum := R.PCIDevsToNUMANode[k]
|
||||||
|
b.WriteString(fmt.Sprintf("PCI %s=%02d\n", k, nodeNum))
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCPUToNUMANodeMapFromEnv(f *framework.Framework, pod *v1.Pod, environ map[string]string, numaNodes int) (map[int]int, error) {
|
||||||
|
var cpuIDs []int
|
||||||
|
cpuListAllowedEnvVar := "CPULIST_ALLOWED"
|
||||||
|
|
||||||
|
for name, value := range environ {
|
||||||
|
if name == cpuListAllowedEnvVar {
|
||||||
|
cpus, err := cpuset.Parse(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cpuIDs = cpus.ToSlice()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(cpuIDs) == 0 {
|
||||||
|
return nil, fmt.Errorf("variable %q found in environ", cpuListAllowedEnvVar)
|
||||||
|
}
|
||||||
|
|
||||||
|
cpusPerNUMA := make(map[int][]int)
|
||||||
|
for numaNode := 0; numaNode < numaNodes; numaNode++ {
|
||||||
|
nodeCPUList := f.ExecCommandInContainer(pod.Name, pod.Spec.Containers[0].Name,
|
||||||
|
"/bin/cat", fmt.Sprintf("/sys/devices/system/node/node%d/cpulist", numaNode))
|
||||||
|
|
||||||
|
cpus, err := cpuset.Parse(nodeCPUList)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cpusPerNUMA[numaNode] = cpus.ToSlice()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CPU IDs -> NUMA Node ID
|
||||||
|
CPUToNUMANode := make(map[int]int)
|
||||||
|
for nodeNum, cpus := range cpusPerNUMA {
|
||||||
|
for _, cpu := range cpus {
|
||||||
|
CPUToNUMANode[cpu] = nodeNum
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// filter out only the allowed CPUs
|
||||||
|
CPUMap := make(map[int]int)
|
||||||
|
for _, cpuID := range cpuIDs {
|
||||||
|
_, ok := CPUToNUMANode[cpuID]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("CPU %d not found on NUMA map: %v", cpuID, CPUToNUMANode)
|
||||||
|
}
|
||||||
|
CPUMap[cpuID] = CPUToNUMANode[cpuID]
|
||||||
|
}
|
||||||
|
return CPUMap, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getPCIDeviceToNumaNodeMapFromEnv(f *framework.Framework, pod *v1.Pod, environ map[string]string) (map[string]int, error) {
|
||||||
|
pciDevPrefix := "PCIDEVICE_"
|
||||||
|
// at this point we don't care which plugin selected the device,
|
||||||
|
// we only need to know which devices were assigned to the POD.
|
||||||
|
// Hence, do prefix search for the variable and fetch the device(s).
|
||||||
|
|
||||||
|
NUMAPerDev := make(map[string]int)
|
||||||
|
for name, value := range environ {
|
||||||
|
if !strings.HasPrefix(name, pciDevPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// a single plugin can allocate more than a single device
|
||||||
|
pciDevs := strings.Split(value, ",")
|
||||||
|
for _, pciDev := range pciDevs {
|
||||||
|
pciDevNUMANode := f.ExecCommandInContainer(pod.Name, pod.Spec.Containers[0].Name,
|
||||||
|
"/bin/cat", fmt.Sprintf("/sys/bus/pci/devices/%s/numa_node", pciDev))
|
||||||
|
|
||||||
|
nodeNum, err := strconv.Atoi(pciDevNUMANode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
NUMAPerDev[pciDev] = nodeNum
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(NUMAPerDev) == 0 {
|
||||||
|
return nil, fmt.Errorf("no PCI devices found in environ")
|
||||||
|
}
|
||||||
|
return NUMAPerDev, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeEnvMap(logs string) (map[string]string, error) {
|
||||||
|
podEnv := strings.Split(logs, "\n")
|
||||||
|
envMap := make(map[string]string)
|
||||||
|
for _, envVar := range podEnv {
|
||||||
|
if len(envVar) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pair := strings.SplitN(envVar, "=", 2)
|
||||||
|
if len(pair) != 2 {
|
||||||
|
return nil, fmt.Errorf("unable to split %q", envVar)
|
||||||
|
}
|
||||||
|
envMap[pair[0]] = pair[1]
|
||||||
|
}
|
||||||
|
return envMap, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkNUMAAlignment(f *framework.Framework, pod *v1.Pod, logs string, numaNodes int) (numaPodResources, error) {
|
||||||
|
podEnv, err := makeEnvMap(logs)
|
||||||
|
if err != nil {
|
||||||
|
return numaPodResources{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
CPUToNUMANode, err := getCPUToNUMANodeMapFromEnv(f, pod, podEnv, numaNodes)
|
||||||
|
if err != nil {
|
||||||
|
return numaPodResources{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
PCIDevsToNUMANode, err := getPCIDeviceToNumaNodeMapFromEnv(f, pod, podEnv)
|
||||||
|
if err != nil {
|
||||||
|
return numaPodResources{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
numaRes := numaPodResources{
|
||||||
|
CPUToNUMANode: CPUToNUMANode,
|
||||||
|
PCIDevsToNUMANode: PCIDevsToNUMANode,
|
||||||
|
}
|
||||||
|
aligned := numaRes.CheckAlignment()
|
||||||
|
if !aligned {
|
||||||
|
return numaRes, fmt.Errorf("NUMA resources not aligned")
|
||||||
|
}
|
||||||
|
return numaRes, nil
|
||||||
|
}
|
@ -43,6 +43,10 @@ import (
|
|||||||
"github.com/onsi/gomega"
|
"github.com/onsi/gomega"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
numalignCmd = `export CPULIST_ALLOWED=$( awk -F":\t*" '/Cpus_allowed_list/ { print $2 }' /proc/self/status); env; sleep 1d`
|
||||||
|
)
|
||||||
|
|
||||||
// Helper for makeTopologyManagerPod().
|
// Helper for makeTopologyManagerPod().
|
||||||
type tmCtnAttribute struct {
|
type tmCtnAttribute struct {
|
||||||
ctnName string
|
ctnName string
|
||||||
@ -222,6 +226,18 @@ func deletePodInNamespace(f *framework.Framework, namespace, name string) {
|
|||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validatePodAlignment(f *framework.Framework, pod *v1.Pod, numaNodes int) {
|
||||||
|
ginkgo.By("validating the Gu pod")
|
||||||
|
logs, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
|
||||||
|
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
|
||||||
|
pod.Spec.Containers[0].Name, pod.Name)
|
||||||
|
|
||||||
|
framework.Logf("got pod logs: %v", logs)
|
||||||
|
numaRes, err := checkNUMAAlignment(f, pod, logs, numaNodes)
|
||||||
|
framework.ExpectNoError(err, "NUMA Alignment check failed for [%s] of pod [%s]: %s",
|
||||||
|
pod.Spec.Containers[0].Name, pod.Name, numaRes.String())
|
||||||
|
}
|
||||||
|
|
||||||
func runTopologyManagerPolicySuiteTests(f *framework.Framework) {
|
func runTopologyManagerPolicySuiteTests(f *framework.Framework) {
|
||||||
var cpuCap, cpuAlloc int64
|
var cpuCap, cpuAlloc int64
|
||||||
var cpuListString, expAllowedCPUsListRegex string
|
var cpuListString, expAllowedCPUsListRegex string
|
||||||
@ -448,7 +464,29 @@ func runTopologyManagerPolicySuiteTests(f *framework.Framework) {
|
|||||||
waitForContainerRemoval(pod2.Spec.Containers[0].Name, pod2.Name, pod2.Namespace)
|
waitForContainerRemoval(pod2.Spec.Containers[0].Name, pod2.Name, pod2.Namespace)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runTopologyManagerNodeAlignmentSuiteTests(f *framework.Framework) {
|
func runTopologyManagerNodeAlignmentSinglePodTest(f *framework.Framework, numaNodes int) {
|
||||||
|
ginkgo.By("allocate aligned resources for a single pod")
|
||||||
|
ctnAttrs := []tmCtnAttribute{
|
||||||
|
{
|
||||||
|
ctnName: "gu-container",
|
||||||
|
cpuRequest: "1000m",
|
||||||
|
cpuLimit: "1000m",
|
||||||
|
devResource: SRIOVResourceName,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pod := makeTopologyManagerTestPod("gu-pod", numalignCmd, ctnAttrs)
|
||||||
|
pod = f.PodClient().CreateSync(pod)
|
||||||
|
|
||||||
|
validatePodAlignment(f, pod, numaNodes)
|
||||||
|
|
||||||
|
framework.Logf("deleting the pod %s/%s and waiting for container %s removal",
|
||||||
|
pod.Namespace, pod.Name, pod.Spec.Containers[0].Name)
|
||||||
|
deletePods(f, []string{pod.Name})
|
||||||
|
waitForContainerRemoval(pod.Spec.Containers[0].Name, pod.Name, pod.Namespace)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runTopologyManagerNodeAlignmentSuiteTests(f *framework.Framework, numaNodes int) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
configMap := readConfigMapV1OrDie(testfiles.ReadOrDie(SRIOVDevicePluginCMYAML))
|
configMap := readConfigMapV1OrDie(testfiles.ReadOrDie(SRIOVDevicePluginCMYAML))
|
||||||
@ -480,27 +518,7 @@ func runTopologyManagerNodeAlignmentSuiteTests(f *framework.Framework) {
|
|||||||
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrue())
|
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrue())
|
||||||
framework.Logf("Successfully created device plugin pod")
|
framework.Logf("Successfully created device plugin pod")
|
||||||
|
|
||||||
ginkgo.By("running a Gu pod")
|
runTopologyManagerNodeAlignmentSinglePodTest(f, numaNodes)
|
||||||
ctnAttrs := []tmCtnAttribute{
|
|
||||||
{
|
|
||||||
ctnName: "gu-container",
|
|
||||||
cpuRequest: "1000m",
|
|
||||||
cpuLimit: "1000m",
|
|
||||||
devResource: SRIOVResourceName,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
pod := makeTopologyManagerTestPod("gu-pod", "env && sleep 1d", ctnAttrs)
|
|
||||||
pod = f.PodClient().CreateSync(pod)
|
|
||||||
|
|
||||||
ginkgo.By("validating the Gu pod")
|
|
||||||
_, err = e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
|
|
||||||
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
|
|
||||||
pod.Spec.Containers[0].Name, pod.Name)
|
|
||||||
|
|
||||||
ginkgo.By("by deleting the pods and waiting for container removal")
|
|
||||||
deletePods(f, []string{pod.Name})
|
|
||||||
waitForContainerRemoval(pod.Spec.Containers[0].Name, pod.Name, pod.Namespace)
|
|
||||||
|
|
||||||
framework.Logf("deleting the SRIOV device plugin pod %s/%s and waiting for container %s removal",
|
framework.Logf("deleting the SRIOV device plugin pod %s/%s and waiting for container %s removal",
|
||||||
dpPod.Namespace, dpPod.Name, dpPod.Spec.Containers[0].Name)
|
dpPod.Namespace, dpPod.Name, dpPod.Spec.Containers[0].Name)
|
||||||
@ -561,7 +579,7 @@ func runTopologyManagerTests(f *framework.Framework) {
|
|||||||
|
|
||||||
configureTopologyManagerInKubelet(f, oldCfg, policy)
|
configureTopologyManagerInKubelet(f, oldCfg, policy)
|
||||||
|
|
||||||
runTopologyManagerNodeAlignmentSuiteTests(f)
|
runTopologyManagerNodeAlignmentSuiteTests(f, numaNodes)
|
||||||
|
|
||||||
// restore kubelet config
|
// restore kubelet config
|
||||||
setOldKubeletConfig(f, oldCfg)
|
setOldKubeletConfig(f, oldCfg)
|
||||||
|
Loading…
Reference in New Issue
Block a user