github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

e2e_node: refactor code to use a single method to update the kubelet config

Browse Source 
Signed-off-by: Artyom Lukianov <alukiano@redhat.com>
This commit is contained in:
Artyom Lukianov 2021-11-04 15:24:44 +02:00
parent ca35bdb403
commit 50fdcdfc59
7 changed files with 380 additions and 631 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
test/e2e_node/cpu_manager_test.go
View File

@ -38,10 +38,8 @@ import (
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
)
// Helper for makeCPUManagerPod().
@ -186,60 +184,15 @@ func getCoreSiblingList(cpuRes int64) string {
return string(out)
}
func deleteStateFile() {
err := exec.Command("/bin/sh", "-c", "rm -f /var/lib/kubelet/cpu_manager_state").Run()
framework.ExpectNoError(err, "error deleting state file")
}
func setOldKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) {
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
// Delete the CPU Manager state file so that the old Kubelet configuration
// can take effect
deleteStateFile()
if oldCfg != nil {
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(oldCfg))
}
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
}
type cpuManagerKubeletArguments struct {
policyName string
cleanStateFile bool
enableCPUManager bool
enableCPUManagerOptions bool
reservedSystemCPUs cpuset.CPUSet
options map[string]string
}
func disableCPUManagerInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.KubeletConfiguration) {
// Disable CPU Manager in Kubelet.
oldCfg = configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
enableCPUManager: false,
})
return oldCfg
}
func configureCPUManagerInKubelet(f *framework.Framework, kubeletArguments *cpuManagerKubeletArguments) (oldCfg *kubeletconfig.KubeletConfiguration) {
// Enable CPU Manager in Kubelet with static policy.
oldCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *cpuManagerKubeletArguments) *kubeletconfig.KubeletConfiguration {
newCfg := oldCfg.DeepCopy()
if newCfg.FeatureGates == nil {
newCfg.FeatureGates = make(map[string]bool)
@ -275,44 +228,6 @@ func configureCPUManagerInKubelet(f *framework.Framework, kubeletArguments *cpuM
}
}
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
// After graduation of the CPU Manager feature to Beta, the CPU Manager
// "none" policy is ON by default. But when we set the CPU Manager policy to
// "static" in this test and the Kubelet is restarted so that "static"
// policy can take effect, there will always be a conflict with the state
// checkpointed in the disk (i.e., the policy checkpointed in the disk will
// be "none" whereas we are trying to restart Kubelet with "static"
// policy). Therefore, we delete the state file so that we can proceed
// with the tests.
// Only delete the state file at the begin of the tests.
if kubeletArguments.cleanStateFile {
deleteStateFile()
}
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
return oldCfg
}
@ -615,6 +530,12 @@ func runCPUManagerTests(f *framework.Framework) {
var ctnAttrs []ctnAttribute
var pod *v1.Pod
ginkgo.BeforeEach(func() {
var err error
oldCfg, err = getCurrentKubeletConfig()
framework.ExpectNoError(err)
})
ginkgo.It("should assign CPUs as expected based on the Pod spec", func() {
cpuCap, cpuAlloc, _ = getLocalNodeCPUDetails(f)
@ -624,12 +545,12 @@ func runCPUManagerTests(f *framework.Framework) {
}
// Enable CPU Manager in the kubelet.
oldCfg = configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
enableCPUManager: true,
reservedSystemCPUs: cpuset.CPUSet{},
cleanStateFile: true,
})
updateKubeletConfig(f, newCfg, true)
ginkgo.By("running a non-Gu pod")
runNonGuPodTest(f, cpuCap)
@ -689,18 +610,24 @@ func runCPUManagerTests(f *framework.Framework) {
pod.Spec.Containers[0].Name, pod.Name)
ginkgo.By("disable cpu manager in kubelet")
disableCPUManagerInKubelet(f)
newCfg = configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
enableCPUManager: false,
reservedSystemCPUs: cpuset.CPUSet{},
})
updateKubeletConfig(f, newCfg, false)
ginkgo.By("by deleting the pod and waiting for container removal")
deletePods(f, []string{pod.Name})
waitForContainerRemoval(pod.Spec.Containers[0].Name, pod.Name, pod.Namespace)
ginkgo.By("enable cpu manager in kubelet without delete state file")
configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
newCfg = configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
enableCPUManager: true,
reservedSystemCPUs: cpuset.CPUSet{},
})
updateKubeletConfig(f, newCfg, false)
ginkgo.By("wait for the deleted pod to be cleaned up from the state file")
waitForStateFileCleanedUp()
@ -729,15 +656,16 @@ func runCPUManagerTests(f *framework.Framework) {
cpuPolicyOptions := map[string]string{
cpumanager.FullPCPUsOnlyOption: "true",
}
oldCfg = configureCPUManagerInKubelet(f,
newCfg := configureCPUManagerInKubelet(oldCfg,
&cpuManagerKubeletArguments{
policyName: string(cpumanager.PolicyStatic),
enableCPUManager: true,
cleanStateFile: true,
reservedSystemCPUs: cpuset.NewCPUSet(0),
enableCPUManagerOptions: true,
options: cpuPolicyOptions,
})
},
)
updateKubeletConfig(f, newCfg, true)
// the order between negative and positive doesn't really matter
runSMTAlignmentNegativeTests(f)
@ -745,7 +673,7 @@ func runCPUManagerTests(f *framework.Framework) {
})
ginkgo.AfterEach(func() {
setOldKubeletConfig(oldCfg)
updateKubeletConfig(f, oldCfg, true)
})
}

18
test/e2e_node/device_manager_test.go
View File

@ -61,15 +61,6 @@ var _ = SIGDescribe("Device Manager [Serial] [Feature:DeviceManager][NodeFeatur
e2eskipper.Skipf("this test is meant to run on a system with at least one configured VF from SRIOV device")
}
oldCfg := enablePodResourcesFeatureGateInKubelet(f)
defer func() {
// restore kubelet config
setOldKubeletConfig(f, oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
sd := setupSRIOVConfigOrFail(f, configMap)
@ -167,15 +158,6 @@ var _ = SIGDescribe("Device Manager [Serial] [Feature:DeviceManager][NodeFeatur
e2eskipper.Skipf("this test is meant to run on a system with at least one configured VF from SRIOV device")
}
oldCfg := enablePodResourcesFeatureGateInKubelet(f)
defer func() {
// restore kubelet config
setOldKubeletConfig(f, oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)

1
test/e2e_node/device_plugin_test.go
View File

@ -110,7 +110,6 @@ func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
pluginSockDir = filepath.Join(pluginSockDir) + "/"
ginkgo.Context("DevicePlugin", func() {
ginkgo.By("Enabling support for Kubelet Plugins Watcher")
ginkgo.It("[Flaky] Verifies the Kubelet device plugin functionality.", func() {
ginkgo.By("Wait for node is ready to start with")
e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)

119
test/e2e_node/memory_manager_test.go
View File

@ -41,9 +41,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state"
"k8s.io/kubernetes/pkg/kubelet/util"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
"k8s.io/utils/pointer"
"github.com/onsi/ginkgo"
@ -52,7 +50,6 @@ import (
const (
evictionHardMemory = "memory.available"
memoryManagerStateFile = "/var/lib/kubelet/memory_manager_state"
resourceMemory = "memory"
staticPolicy = "Static"
nonePolicy = "None"
@ -138,11 +135,6 @@ func makeMemoryManagerPod(podName string, initCtnAttributes, ctnAttributes []mem
return pod
}
func deleteMemoryManagerStateFile() {
err := exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", memoryManagerStateFile)).Run()
framework.ExpectNoError(err, "failed to delete the state file")
}
func getMemoryManagerState() (*state.MemoryManagerCheckpoint, error) {
if _, err := os.Stat(memoryManagerStateFile); os.IsNotExist(err) {
return nil, fmt.Errorf("the memory manager state file %s does not exist", memoryManagerStateFile)
@ -187,76 +179,44 @@ type kubeletParams struct {
evictionHard map[string]string
}
func getUpdatedKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration, params *kubeletParams) *kubeletconfig.KubeletConfiguration {
newCfg := oldCfg.DeepCopy()
if newCfg.FeatureGates == nil {
newCfg.FeatureGates = map[string]bool{}
func updateKubeletConfigWithMemoryManagerParams(initialCfg *kubeletconfig.KubeletConfiguration, params *kubeletParams) {
if initialCfg.FeatureGates == nil {
initialCfg.FeatureGates = map[string]bool{}
}
newCfg.MemoryManagerPolicy = params.memoryManagerPolicy
initialCfg.MemoryManagerPolicy = params.memoryManagerPolicy
// update system-reserved
if newCfg.SystemReserved == nil {
newCfg.SystemReserved = map[string]string{}
if initialCfg.SystemReserved == nil {
initialCfg.SystemReserved = map[string]string{}
}
for resourceName, value := range params.systemReserved {
newCfg.SystemReserved[resourceName] = value
initialCfg.SystemReserved[resourceName] = value
}
// update kube-reserved
if newCfg.KubeReserved == nil {
newCfg.KubeReserved = map[string]string{}
if initialCfg.KubeReserved == nil {
initialCfg.KubeReserved = map[string]string{}
}
for resourceName, value := range params.kubeReserved {
newCfg.KubeReserved[resourceName] = value
initialCfg.KubeReserved[resourceName] = value
}
// update hard eviction threshold
if newCfg.EvictionHard == nil {
newCfg.EvictionHard = map[string]string{}
if initialCfg.EvictionHard == nil {
initialCfg.EvictionHard = map[string]string{}
}
for resourceName, value := range params.evictionHard {
newCfg.EvictionHard[resourceName] = value
initialCfg.EvictionHard[resourceName] = value
}
// update reserved memory
if newCfg.ReservedMemory == nil {
newCfg.ReservedMemory = []kubeletconfig.MemoryReservation{}
if initialCfg.ReservedMemory == nil {
initialCfg.ReservedMemory = []kubeletconfig.MemoryReservation{}
}
for _, memoryReservation := range params.systemReservedMemory {
newCfg.ReservedMemory = append(newCfg.ReservedMemory, memoryReservation)
initialCfg.ReservedMemory = append(initialCfg.ReservedMemory, memoryReservation)
}
return newCfg
}
func updateKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(cfg))
deleteMemoryManagerStateFile()
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
}
func getAllNUMANodes() []int {
@ -288,8 +248,6 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
ctnParams, initCtnParams []memoryManagerCtnAttributes
is2MiHugepagesSupported *bool
isMultiNUMASupported *bool
kubeParams *kubeletParams
oldCfg *kubeletconfig.KubeletConfiguration
testPod *v1.Pod
)
@ -364,7 +322,6 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
// dynamically update the kubelet configuration
ginkgo.JustBeforeEach(func() {
var err error
hugepagesCount := 8
// allocate hugepages
@ -373,22 +330,9 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
gomega.Eventually(func() error {
return configureHugePages(hugepagesSize2M, hugepagesCount)
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
}
// get the old kubelet config
if oldCfg == nil {
gomega.Eventually(func() error {
oldCfg, err = e2enodekubelet.GetCurrentKubeletConfigFromFile()
return err
}, 5*time.Minute, 15*time.Second).Should(gomega.BeNil())
}
restartKubelet(true)
// update the kubelet config with new parameters
newCfg := getUpdatedKubeletConfig(oldCfg, kubeParams)
updateKubeletConfig(f, newCfg)
// request hugepages resources under the container
if *is2MiHugepagesSupported {
ginkgo.By("Waiting for hugepages resource to become available on the local node")
waitingForHugepages(hugepagesCount)
@ -414,24 +358,18 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
gomega.Eventually(func() error {
return configureHugePages(hugepagesSize2M, 0)
}, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages")
}
// update the kubelet config with old parameters
if oldCfg != nil {
updateKubeletConfig(f, oldCfg)
}
restartKubelet(true)
if *is2MiHugepagesSupported {
waitingForHugepages(0)
}
})
ginkgo.Context("with static policy", func() {
ginkgo.BeforeEach(func() {
// override kubelet configuration parameters
tmpParams := *defaultKubeParams
tmpParams.memoryManagerPolicy = staticPolicy
kubeParams = &tmpParams
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
kubeParams := *defaultKubeParams
kubeParams.memoryManagerPolicy = staticPolicy
updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
})
ginkgo.JustAfterEach(func() {
@ -706,11 +644,15 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
})
ginkgo.Context("with none policy", func() {
ginkgo.BeforeEach(func() {
tmpParams := *defaultKubeParams
tmpParams.memoryManagerPolicy = nonePolicy
kubeParams = &tmpParams
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
kubeParams := *defaultKubeParams
kubeParams.memoryManagerPolicy = nonePolicy
updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
})
// empty context to configure same container parameters for all tests
ginkgo.Context("", func() {
ginkgo.BeforeEach(func() {
// override pod parameters
ctnParams = []memoryManagerCtnAttributes{
{
@ -772,4 +714,5 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
verifyMemoryPinning(testPod, allNUMANodes)
})
})
})
})

269
test/e2e_node/podresources_test.go
View File

@ -28,7 +28,6 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
kubefeatures "k8s.io/kubernetes/pkg/features"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
@ -38,15 +37,13 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util"
testutils "k8s.io/kubernetes/test/utils"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
type podDesc struct {
@ -536,30 +533,39 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
reservedSystemCPUs := cpuset.MustParse("1")
ginkgo.Context("With SRIOV devices in the system", func() {
ginkgo.It("should return the expected responses with cpumanager static policy enabled", func() {
ginkgo.Context("with SRIOV devices in the system", func() {
ginkgo.BeforeEach(func() {
requireSRIOVDevices()
})
ginkgo.Context("with CPU manager Static policy", func() {
ginkgo.BeforeEach(func() {
// this is a very rough check. We just want to rule out system that does NOT have enough resources
_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
if cpuAlloc < minCoreCount {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
}
})
requireSRIOVDevices()
// empty context to apply kubelet config changes
ginkgo.Context("", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
// Set the CPU Manager policy to static.
initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Set the CPU Manager reconcile period to 1 second.
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
initialConfig.ReservedSystemCPUs = cpus
})
ginkgo.It("should return the expected responses", func() {
onlineCPUs, err := getOnlineCPUs()
framework.ExpectNoError(err)
// Make sure all the feature gates and the right settings are in place.
oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
defer func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
sd := setupSRIOVConfigOrFail(f, configMap)
defer teardownSRIOVConfigOrFail(f, sd)
@ -579,23 +585,16 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
podresourcesListTests(f, cli, sd)
ginkgo.By("checking GetAllocatableResources()")
podresourcesGetAllocatableResourcesTests(cli, sd, onlineCPUs, reservedSystemCPUs)
})
})
})
ginkgo.It("should return the expected responses with cpumanager none policy", func() {
ginkgo.Context("with CPU manager None policy", func() {
ginkgo.It("should return the expected responses", func() {
// current default is "none" policy - no need to restart the kubelet
requireSRIOVDevices()
oldCfg := enablePodResourcesFeatureGateInKubelet(f)
defer func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
sd := setupSRIOVConfigOrFail(f, configMap)
defer teardownSRIOVConfigOrFail(f, sd)
@ -615,33 +614,42 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
podresourcesGetAllocatableResourcesTests(cli, sd, cpuset.CPUSet{}, cpuset.CPUSet{})
})
})
})
ginkgo.Context("Without SRIOV devices in the system", func() {
ginkgo.It("should return the expected responses with cpumanager static policy enabled", func() {
ginkgo.Context("without SRIOV devices in the system", func() {
ginkgo.BeforeEach(func() {
requireLackOfSRIOVDevices()
})
ginkgo.Context("with CPU manager Static policy", func() {
ginkgo.BeforeEach(func() {
// this is a very rough check. We just want to rule out system that does NOT have enough resources
_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
if cpuAlloc < minCoreCount {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
}
})
requireLackOfSRIOVDevices()
// empty context to apply kubelet config changes
ginkgo.Context("", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
// Set the CPU Manager policy to static.
initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Set the CPU Manager reconcile period to 1 second.
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
initialConfig.ReservedSystemCPUs = cpus
})
ginkgo.It("should return the expected responses", func() {
onlineCPUs, err := getOnlineCPUs()
framework.ExpectNoError(err)
// Make sure all the feature gates and the right settings are in place.
oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
defer func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
@ -652,21 +660,11 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
podresourcesListTests(f, cli, nil)
podresourcesGetAllocatableResourcesTests(cli, nil, onlineCPUs, reservedSystemCPUs)
})
})
})
ginkgo.It("should return the expected responses with cpumanager none policy", func() {
// current default is "none" policy - no need to restart the kubelet
requireLackOfSRIOVDevices()
oldCfg := enablePodResourcesFeatureGateInKubelet(f)
defer func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
// Delete state file to allow repeated runs
deleteStateFile()
}()
ginkgo.Context("with CPU manager None policy", func() {
ginkgo.It("should return the expected responses", func() {
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
@ -678,10 +676,17 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
podresourcesGetAllocatableResourcesTests(cli, nil, cpuset.CPUSet{}, cpuset.CPUSet{})
})
})
ginkgo.Context("with disabled KubeletPodResourcesGetAllocatable feature gate", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
if initialConfig.FeatureGates == nil {
initialConfig.FeatureGates = make(map[string]bool)
}
initialConfig.FeatureGates[string(kubefeatures.KubeletPodResourcesGetAllocatable)] = false
})
ginkgo.It("should return the expected error with the feature gate disabled", func() {
e2eskipper.SkipIfFeatureGateEnabled(kubefeatures.KubeletPodResourcesGetAllocatable)
endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
framework.ExpectNoError(err)
@ -695,35 +700,41 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
framework.ExpectError(err, "With feature gate disabled, the call must fail")
})
})
})
ginkgo.Context("Use KubeVirt device plugin, which reports resources w/o hardware topology", func() {
ginkgo.It("should return proper podresources the same as before the restart of kubelet", func() {
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResources) {
e2eskipper.Skipf("this test is meant to run with the POD Resources Extensions feature gate enabled")
}
ginkgo.Context("with KubeVirt device plugin, which reports resources w/o hardware topology", func() {
ginkgo.BeforeEach(func() {
_, err := os.Stat("/dev/kvm")
if errors.Is(err, os.ErrNotExist) {
e2eskipper.Skipf("KubeVirt device plugin could work only in kvm based environment")
}
})
ginkgo.Context("with CPU manager Static policy", func() {
ginkgo.BeforeEach(func() {
// this is a very rough check. We just want to rule out system that does NOT have enough resources
_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
if cpuAlloc < minCoreCount {
e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
}
})
// Make sure all the feature gates and the right settings are in place.
oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
defer func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
// empty context to apply kubelet config changes
ginkgo.Context("", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
// Set the CPU Manager policy to static.
initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Delete state file to allow repeated runs
deleteStateFile()
}()
// Set the CPU Manager reconcile period to 1 second.
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
initialConfig.ReservedSystemCPUs = cpus
})
ginkgo.It("should return proper podresources the same as before the restart of kubelet", func() {
dpPod := setupKubeVirtDevicePluginOrFail(f)
defer teardownKubeVirtDevicePluginOrFail(f, dpPod)
@ -769,7 +780,8 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
expectPodResources(1, cli, []podDesc{desc})
tpd.deletePodsForTest(f)
})
})
})
})
})
@ -787,113 +799,6 @@ func getOnlineCPUs() (cpuset.CPUSet, error) {
return cpuset.Parse(strings.TrimSpace(string(onlineCPUList)))
}
func configurePodResourcesInKubelet(f *framework.Framework, cleanStateFile bool, reservedSystemCPUs cpuset.CPUSet) (oldCfg *kubeletconfig.KubeletConfiguration) {
// we also need CPUManager with static policy to be able to do meaningful testing
oldCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
newCfg := oldCfg.DeepCopy()
newCfg.FeatureGates[string(kubefeatures.KubeletPodResourcesGetAllocatable)] = true
// After graduation of the CPU Manager feature to Beta, the CPU Manager
// "none" policy is ON by default. But when we set the CPU Manager policy to
// "static" in this test and the Kubelet is restarted so that "static"
// policy can take effect, there will always be a conflict with the state
// checkpointed in the disk (i.e., the policy checkpointed in the disk will
// be "none" whereas we are trying to restart Kubelet with "static"
// policy). Therefore, we delete the state file so that we can proceed
// with the tests.
// Only delete the state file at the begin of the tests.
if cleanStateFile {
deleteStateFile()
}
// Set the CPU Manager policy to static.
newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Set the CPU Manager reconcile period to 1 second.
newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
if reservedSystemCPUs.Size() > 0 {
cpus := reservedSystemCPUs.String()
framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
newCfg.ReservedSystemCPUs = cpus
} else {
// The Kubelet panics if either kube-reserved or system-reserved is not set
// when CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
// kubelet doesn't panic.
if newCfg.KubeReserved == nil {
newCfg.KubeReserved = map[string]string{}
}
if _, ok := newCfg.KubeReserved["cpu"]; !ok {
newCfg.KubeReserved["cpu"] = "200m"
}
}
// Update the Kubelet configuration.
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
return oldCfg
}
func enablePodResourcesFeatureGateInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.KubeletConfiguration) {
oldCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
newCfg := oldCfg.DeepCopy()
if newCfg.FeatureGates == nil {
newCfg.FeatureGates = make(map[string]bool)
}
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
return oldCfg
}
func setupKubeVirtDevicePluginOrFail(f *framework.Framework) *v1.Pod {
e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)

38
test/e2e_node/topology_manager_test.go
View File

@ -42,7 +42,6 @@ import (
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
testutils "k8s.io/kubernetes/test/utils"
"github.com/onsi/ginkgo"
@ -194,7 +193,7 @@ func findNUMANodeWithoutSRIOVDevices(configMap *v1.ConfigMap, numaNodes int) (in
return findNUMANodeWithoutSRIOVDevicesFromSysfs(numaNodes)
}
func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletconfig.KubeletConfiguration, policy, scope string, configMap *v1.ConfigMap, numaNodes int) string {
func configureTopologyManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, policy, scope string, configMap *v1.ConfigMap, numaNodes int) (*kubeletconfig.KubeletConfiguration, string) {
// Configure Topology Manager in Kubelet with policy.
newCfg := oldCfg.DeepCopy()
if newCfg.FeatureGates == nil {
@ -204,8 +203,6 @@ func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletco
newCfg.FeatureGates["CPUManager"] = true
newCfg.FeatureGates["TopologyManager"] = true
deleteStateFile()
// Set the Topology Manager policy
newCfg.TopologyManagerPolicy = policy
@ -237,27 +234,7 @@ func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletco
// Dump the config -- debug
framework.Logf("New kubelet config is %s", *newCfg)
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
ginkgo.By("Starting the kubelet")
startKubelet()
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
return newCfg.ReservedSystemCPUs
return newCfg, newCfg.ReservedSystemCPUs
}
// getSRIOVDevicePluginPod returns the Device Plugin pod for sriov resources in e2e tests.
@ -910,7 +887,8 @@ func runTopologyManagerTests(f *framework.Framework) {
ginkgo.By(fmt.Sprintf("by configuring Topology Manager policy to %s", policy))
framework.Logf("Configuring topology Manager policy to %s", policy)
configureTopologyManagerInKubelet(f, oldCfg, policy, scope, nil, 0)
newCfg, _ := configureTopologyManagerInKubelet(oldCfg, policy, scope, nil, 0)
updateKubeletConfig(f, newCfg, true)
// Run the tests
runTopologyManagerPolicySuiteTests(f)
}
@ -933,7 +911,8 @@ func runTopologyManagerTests(f *framework.Framework) {
ginkgo.By(fmt.Sprintf("by configuring Topology Manager policy to %s", policy))
framework.Logf("Configuring topology Manager policy to %s", policy)
reservedSystemCPUs := configureTopologyManagerInKubelet(f, oldCfg, policy, scope, configMap, numaNodes)
newCfg, reservedSystemCPUs := configureTopologyManagerInKubelet(oldCfg, policy, scope, configMap, numaNodes)
updateKubeletConfig(f, newCfg, true)
runTopologyManagerNodeAlignmentSuiteTests(f, sd, reservedSystemCPUs, policy, numaNodes, coreCount)
}
@ -950,14 +929,15 @@ func runTopologyManagerTests(f *framework.Framework) {
policy := topologymanager.PolicySingleNumaNode
scope := podScopeTopology
reservedSystemCPUs := configureTopologyManagerInKubelet(f, oldCfg, policy, scope, configMap, numaNodes)
newCfg, reservedSystemCPUs := configureTopologyManagerInKubelet(oldCfg, policy, scope, configMap, numaNodes)
updateKubeletConfig(f, newCfg, true)
runTMScopeResourceAlignmentTestSuite(f, configMap, reservedSystemCPUs, policy, numaNodes, coreCount)
})
ginkgo.AfterEach(func() {
// restore kubelet config
setOldKubeletConfig(oldCfg)
updateKubeletConfig(f, oldCfg, true)
})
}

54
test/e2e_node/util.go
View File

@ -75,6 +75,9 @@ const (
defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
kubeletReadOnlyPort = "10255"
kubeletHealthCheckURL = "http://127.0.0.1:" + kubeletReadOnlyPort + "/healthz"
// state files
cpuManagerStateFile = "/var/lib/kubelet/cpu_manager_state"
memoryManagerStateFile = "/var/lib/kubelet/memory_manager_state"
)
func getNodeSummary() (*stats.Summary, error) {
@ -158,36 +161,29 @@ func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) {
// Returns true on success.
func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(initialConfig *kubeletconfig.KubeletConfiguration)) {
var oldCfg *kubeletconfig.KubeletConfiguration
ginkgo.BeforeEach(func() {
oldCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
newCfg := oldCfg.DeepCopy()
updateFunction(newCfg)
if apiequality.Semantic.DeepEqual(*newCfg, *oldCfg) {
return
}
// Update the Kubelet configuration.
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
ginkgo.By("Starting the kubelet")
startKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
updateKubeletConfig(f, newCfg, true)
})
ginkgo.AfterEach(func() {
if oldCfg != nil {
// Update the Kubelet configuration.
updateKubeletConfig(f, oldCfg, true)
}
})
}
func updateKubeletConfig(f *framework.Framework, kubeletConfig *kubeletconfig.KubeletConfiguration, deleteStateFiles bool) {
// Update the Kubelet configuration.
ginkgo.By("Stopping the kubelet")
startKubelet := stopKubelet()
@ -197,7 +193,13 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini
return kubeletHealthCheck(kubeletHealthCheckURL)
}, time.Minute, time.Second).Should(gomega.BeFalse())
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(oldCfg))
// Delete CPU and memory manager state files to be sure it will not prevent the kubelet restart
if deleteStateFiles {
deleteStateFile(cpuManagerStateFile)
deleteStateFile(memoryManagerStateFile)
}
framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(kubeletConfig))
ginkgo.By("Starting the kubelet")
startKubelet()
@ -206,8 +208,18 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini
gomega.Eventually(func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
}
})
// Wait for the Kubelet to be ready.
gomega.Eventually(func() bool {
nodes, err := e2enode.TotalReady(f.ClientSet)
framework.ExpectNoError(err)
return nodes == 1
}, time.Minute, time.Second).Should(gomega.BeTrue())
}
func deleteStateFile(stateFileName string) {
err := exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", stateFileName)).Run()
framework.ExpectNoError(err, "failed to delete the state file")
}
// Returns true if kubeletConfig is enabled, false otherwise or if we cannot determine if it is.