e2e_node: refactor code to use a single method to update the kubelet config

Signed-off-by: Artyom Lukianov <alukiano@redhat.com>

test/e2e_node/cpu_manager_test.go

@@ -38,10 +38,8 @@ import (
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
 	"k8s.io/kubernetes/test/e2e/framework"
-	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-	e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
 )
 
 // Helper for makeCPUManagerPod().
@@ -186,60 +184,15 @@ func getCoreSiblingList(cpuRes int64) string {
 	return string(out)
 }
 
-func deleteStateFile() {
-	err := exec.Command("/bin/sh", "-c", "rm -f /var/lib/kubelet/cpu_manager_state").Run()
-	framework.ExpectNoError(err, "error deleting state file")
-}
-
-func setOldKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) {
-	ginkgo.By("Stopping the kubelet")
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	// Delete the CPU Manager state file so that the old Kubelet configuration
-	// can take effect
-	deleteStateFile()
-
-	if oldCfg != nil {
-		framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(oldCfg))
-	}
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// wait until the kubelet health check will succeed
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
-}
-
 type cpuManagerKubeletArguments struct {
 	policyName              string
-	cleanStateFile          bool
 	enableCPUManager        bool
 	enableCPUManagerOptions bool
 	reservedSystemCPUs      cpuset.CPUSet
 	options                 map[string]string
 }
 
-func disableCPUManagerInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.KubeletConfiguration) {
+func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *cpuManagerKubeletArguments) *kubeletconfig.KubeletConfiguration {
-	// Disable CPU Manager in Kubelet.
-	oldCfg = configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
-		enableCPUManager: false,
-	})
-
-	return oldCfg
-}
-
-func configureCPUManagerInKubelet(f *framework.Framework, kubeletArguments *cpuManagerKubeletArguments) (oldCfg *kubeletconfig.KubeletConfiguration) {
-	// Enable CPU Manager in Kubelet with static policy.
-	oldCfg, err := getCurrentKubeletConfig()
-	framework.ExpectNoError(err)
-
 	newCfg := oldCfg.DeepCopy()
 	if newCfg.FeatureGates == nil {
 		newCfg.FeatureGates = make(map[string]bool)
@@ -275,44 +228,6 @@ func configureCPUManagerInKubelet(f *framework.Framework, kubeletArguments *cpuM
 		}
 	}
 
-	ginkgo.By("Stopping the kubelet")
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	// After graduation of the CPU Manager feature to Beta, the CPU Manager
-	// "none" policy is ON by default. But when we set the CPU Manager policy to
-	// "static" in this test and the Kubelet is restarted so that "static"
-	// policy can take effect, there will always be a conflict with the state
-	// checkpointed in the disk (i.e., the policy checkpointed in the disk will
-	// be "none" whereas we are trying to restart Kubelet with "static"
-	// policy). Therefore, we delete the state file so that we can proceed
-	// with the tests.
-	// Only delete the state file at the begin of the tests.
-	if kubeletArguments.cleanStateFile {
-		deleteStateFile()
-	}
-
-	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// wait until the kubelet health check will succeed
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
-
-	// Wait for the Kubelet to be ready.
-	gomega.Eventually(func() bool {
-		nodes, err := e2enode.TotalReady(f.ClientSet)
-		framework.ExpectNoError(err)
-		return nodes == 1
-	}, time.Minute, time.Second).Should(gomega.BeTrue())
-
 	return oldCfg
 }
 
@@ -615,6 +530,12 @@ func runCPUManagerTests(f *framework.Framework) {
 	var ctnAttrs []ctnAttribute
 	var pod *v1.Pod
 
+	ginkgo.BeforeEach(func() {
+		var err error
+		oldCfg, err = getCurrentKubeletConfig()
+		framework.ExpectNoError(err)
+	})
+
 	ginkgo.It("should assign CPUs as expected based on the Pod spec", func() {
 		cpuCap, cpuAlloc, _ = getLocalNodeCPUDetails(f)
 
@@ -624,12 +545,12 @@ func runCPUManagerTests(f *framework.Framework) {
 		}
 
 		// Enable CPU Manager in the kubelet.
-		oldCfg = configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
+		newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
 			policyName:         string(cpumanager.PolicyStatic),
 			enableCPUManager:   true,
 			reservedSystemCPUs: cpuset.CPUSet{},
-			cleanStateFile:     true,
 		})
+		updateKubeletConfig(f, newCfg, true)
 
 		ginkgo.By("running a non-Gu pod")
 		runNonGuPodTest(f, cpuCap)
@@ -689,18 +610,24 @@ func runCPUManagerTests(f *framework.Framework) {
 			pod.Spec.Containers[0].Name, pod.Name)
 
 		ginkgo.By("disable cpu manager in kubelet")
-		disableCPUManagerInKubelet(f)
+		newCfg = configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
+			policyName:         string(cpumanager.PolicyStatic),
+			enableCPUManager:   false,
+			reservedSystemCPUs: cpuset.CPUSet{},
+		})
+		updateKubeletConfig(f, newCfg, false)
 
 		ginkgo.By("by deleting the pod and waiting for container removal")
 		deletePods(f, []string{pod.Name})
 		waitForContainerRemoval(pod.Spec.Containers[0].Name, pod.Name, pod.Namespace)
 
 		ginkgo.By("enable cpu manager in kubelet without delete state file")
-		configureCPUManagerInKubelet(f, &cpuManagerKubeletArguments{
+		newCfg = configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{
 			policyName:         string(cpumanager.PolicyStatic),
 			enableCPUManager:   true,
 			reservedSystemCPUs: cpuset.CPUSet{},
 		})
+		updateKubeletConfig(f, newCfg, false)
 
 		ginkgo.By("wait for the deleted pod to be cleaned up from the state file")
 		waitForStateFileCleanedUp()
@@ -729,15 +656,16 @@ func runCPUManagerTests(f *framework.Framework) {
 		cpuPolicyOptions := map[string]string{
 			cpumanager.FullPCPUsOnlyOption: "true",
 		}
-		oldCfg = configureCPUManagerInKubelet(f,
+		newCfg := configureCPUManagerInKubelet(oldCfg,
 			&cpuManagerKubeletArguments{
 				policyName:              string(cpumanager.PolicyStatic),
 				enableCPUManager:        true,
-				cleanStateFile:          true,
 				reservedSystemCPUs:      cpuset.NewCPUSet(0),
 				enableCPUManagerOptions: true,
 				options:                 cpuPolicyOptions,
-			})
+			},
+		)
+		updateKubeletConfig(f, newCfg, true)
 
 		// the order between negative and positive doesn't really matter
 		runSMTAlignmentNegativeTests(f)
@@ -745,7 +673,7 @@ func runCPUManagerTests(f *framework.Framework) {
 	})
 
 	ginkgo.AfterEach(func() {
-		setOldKubeletConfig(oldCfg)
+		updateKubeletConfig(f, oldCfg, true)
 	})
 }
 

test/e2e_node/device_manager_test.go

@@ -61,15 +61,6 @@ var _ = SIGDescribe("Device Manager  [Serial] [Feature:DeviceManager][NodeFeatur
 				e2eskipper.Skipf("this test is meant to run on a system with at least one configured VF from SRIOV device")
 			}
 
-			oldCfg := enablePodResourcesFeatureGateInKubelet(f)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(f, oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 			configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
 			sd := setupSRIOVConfigOrFail(f, configMap)
 
@@ -167,15 +158,6 @@ var _ = SIGDescribe("Device Manager  [Serial] [Feature:DeviceManager][NodeFeatur
 				e2eskipper.Skipf("this test is meant to run on a system with at least one configured VF from SRIOV device")
 			}
 
-			oldCfg := enablePodResourcesFeatureGateInKubelet(f)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(f, oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 			endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
 			framework.ExpectNoError(err)
 

test/e2e_node/device_plugin_test.go

@@ -110,7 +110,6 @@ func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
 func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 	pluginSockDir = filepath.Join(pluginSockDir) + "/"
 	ginkgo.Context("DevicePlugin", func() {
-		ginkgo.By("Enabling support for Kubelet Plugins Watcher")
 		ginkgo.It("[Flaky] Verifies the Kubelet device plugin functionality.", func() {
 			ginkgo.By("Wait for node is ready to start with")
 			e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)

test/e2e_node/memory_manager_test.go

@@ -41,9 +41,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 	"k8s.io/kubernetes/test/e2e/framework"
-	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
 	"k8s.io/utils/pointer"
 
 	"github.com/onsi/ginkgo"
@@ -52,7 +50,6 @@ import (
 
 const (
 	evictionHardMemory = "memory.available"
-	memoryManagerStateFile = "/var/lib/kubelet/memory_manager_state"
 	resourceMemory     = "memory"
 	staticPolicy       = "Static"
 	nonePolicy         = "None"
@@ -138,11 +135,6 @@ func makeMemoryManagerPod(podName string, initCtnAttributes, ctnAttributes []mem
 	return pod
 }
 
-func deleteMemoryManagerStateFile() {
-	err := exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", memoryManagerStateFile)).Run()
-	framework.ExpectNoError(err, "failed to delete the state file")
-}
-
 func getMemoryManagerState() (*state.MemoryManagerCheckpoint, error) {
 	if _, err := os.Stat(memoryManagerStateFile); os.IsNotExist(err) {
 		return nil, fmt.Errorf("the memory manager state file %s does not exist", memoryManagerStateFile)
@@ -187,76 +179,44 @@ type kubeletParams struct {
 	evictionHard         map[string]string
 }
 
-func getUpdatedKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration, params *kubeletParams) *kubeletconfig.KubeletConfiguration {
+func updateKubeletConfigWithMemoryManagerParams(initialCfg *kubeletconfig.KubeletConfiguration, params *kubeletParams) {
-	newCfg := oldCfg.DeepCopy()
+	if initialCfg.FeatureGates == nil {
-
+		initialCfg.FeatureGates = map[string]bool{}
-	if newCfg.FeatureGates == nil {
-		newCfg.FeatureGates = map[string]bool{}
 	}
 
-	newCfg.MemoryManagerPolicy = params.memoryManagerPolicy
+	initialCfg.MemoryManagerPolicy = params.memoryManagerPolicy
 
 	// update system-reserved
-	if newCfg.SystemReserved == nil {
+	if initialCfg.SystemReserved == nil {
-		newCfg.SystemReserved = map[string]string{}
+		initialCfg.SystemReserved = map[string]string{}
 	}
 	for resourceName, value := range params.systemReserved {
-		newCfg.SystemReserved[resourceName] = value
+		initialCfg.SystemReserved[resourceName] = value
 	}
 
 	// update kube-reserved
-	if newCfg.KubeReserved == nil {
+	if initialCfg.KubeReserved == nil {
-		newCfg.KubeReserved = map[string]string{}
+		initialCfg.KubeReserved = map[string]string{}
 	}
 	for resourceName, value := range params.kubeReserved {
-		newCfg.KubeReserved[resourceName] = value
+		initialCfg.KubeReserved[resourceName] = value
 	}
 
 	// update hard eviction threshold
-	if newCfg.EvictionHard == nil {
+	if initialCfg.EvictionHard == nil {
-		newCfg.EvictionHard = map[string]string{}
+		initialCfg.EvictionHard = map[string]string{}
 	}
 	for resourceName, value := range params.evictionHard {
-		newCfg.EvictionHard[resourceName] = value
+		initialCfg.EvictionHard[resourceName] = value
 	}
 
 	// update reserved memory
-	if newCfg.ReservedMemory == nil {
+	if initialCfg.ReservedMemory == nil {
-		newCfg.ReservedMemory = []kubeletconfig.MemoryReservation{}
+		initialCfg.ReservedMemory = []kubeletconfig.MemoryReservation{}
 	}
 	for _, memoryReservation := range params.systemReservedMemory {
-		newCfg.ReservedMemory = append(newCfg.ReservedMemory, memoryReservation)
+		initialCfg.ReservedMemory = append(initialCfg.ReservedMemory, memoryReservation)
 	}
-
-	return newCfg
-}
-
-func updateKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
-	ginkgo.By("Stopping the kubelet")
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(cfg))
-	deleteMemoryManagerStateFile()
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// wait until the kubelet health check will succeed
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
-
-	// Wait for the Kubelet to be ready.
-	gomega.Eventually(func() bool {
-		nodes, err := e2enode.TotalReady(f.ClientSet)
-		framework.ExpectNoError(err)
-		return nodes == 1
-	}, time.Minute, time.Second).Should(gomega.BeTrue())
 }
 
 func getAllNUMANodes() []int {
@@ -288,8 +248,6 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 		ctnParams, initCtnParams []memoryManagerCtnAttributes
 		is2MiHugepagesSupported  *bool
 		isMultiNUMASupported     *bool
-		kubeParams               *kubeletParams
-		oldCfg                   *kubeletconfig.KubeletConfiguration
 		testPod                  *v1.Pod
 	)
 
@@ -364,7 +322,6 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 
 	// dynamically update the kubelet configuration
 	ginkgo.JustBeforeEach(func() {
-		var err error
 		hugepagesCount := 8
 
 		// allocate hugepages
@@ -373,22 +330,9 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 			gomega.Eventually(func() error {
 				return configureHugePages(hugepagesSize2M, hugepagesCount)
 			}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
-		}
 
-		// get the old kubelet config
+			restartKubelet(true)
-		if oldCfg == nil {
-			gomega.Eventually(func() error {
-				oldCfg, err = e2enodekubelet.GetCurrentKubeletConfigFromFile()
-				return err
-			}, 5*time.Minute, 15*time.Second).Should(gomega.BeNil())
-		}
 
-		// update the kubelet config with new parameters
-		newCfg := getUpdatedKubeletConfig(oldCfg, kubeParams)
-		updateKubeletConfig(f, newCfg)
-
-		// request hugepages resources under the container
-		if *is2MiHugepagesSupported {
 			ginkgo.By("Waiting for hugepages resource to become available on the local node")
 			waitingForHugepages(hugepagesCount)
 
@@ -414,24 +358,18 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 			gomega.Eventually(func() error {
 				return configureHugePages(hugepagesSize2M, 0)
 			}, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages")
-		}
 
-		// update the kubelet config with old parameters
+			restartKubelet(true)
-		if oldCfg != nil {
-			updateKubeletConfig(f, oldCfg)
-		}
 
-		if *is2MiHugepagesSupported {
 			waitingForHugepages(0)
 		}
 	})
 
 	ginkgo.Context("with static policy", func() {
-		ginkgo.BeforeEach(func() {
+		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			// override kubelet configuration parameters
+			kubeParams := *defaultKubeParams
-			tmpParams := *defaultKubeParams
+			kubeParams.memoryManagerPolicy = staticPolicy
-			tmpParams.memoryManagerPolicy = staticPolicy
+			updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
-			kubeParams = &tmpParams
 		})
 
 		ginkgo.JustAfterEach(func() {
@@ -706,11 +644,15 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 	})
 
 	ginkgo.Context("with none policy", func() {
-		ginkgo.BeforeEach(func() {
+		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			tmpParams := *defaultKubeParams
+			kubeParams := *defaultKubeParams
-			tmpParams.memoryManagerPolicy = nonePolicy
+			kubeParams.memoryManagerPolicy = nonePolicy
-			kubeParams = &tmpParams
+			updateKubeletConfigWithMemoryManagerParams(initialConfig, &kubeParams)
+		})
 
+		// empty context to configure same container parameters for all tests
+		ginkgo.Context("", func() {
+			ginkgo.BeforeEach(func() {
 				// override pod parameters
 				ctnParams = []memoryManagerCtnAttributes{
 					{
@@ -773,3 +715,4 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
 			})
 		})
 	})
+})

test/e2e_node/podresources_test.go

@@ -28,7 +28,6 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
 	kubefeatures "k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
@@ -38,15 +37,13 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/util"
 	testutils "k8s.io/kubernetes/test/utils"
 
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
-	e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
-
-	"github.com/onsi/ginkgo"
-	"github.com/onsi/gomega"
 )
 
 type podDesc struct {
@@ -536,30 +533,39 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 
 	reservedSystemCPUs := cpuset.MustParse("1")
 
-	ginkgo.Context("With SRIOV devices in the system", func() {
+	ginkgo.Context("with SRIOV devices in the system", func() {
-		ginkgo.It("should return the expected responses with cpumanager static policy enabled", func() {
+		ginkgo.BeforeEach(func() {
+			requireSRIOVDevices()
+		})
+
+		ginkgo.Context("with CPU manager Static policy", func() {
+			ginkgo.BeforeEach(func() {
 				// this is a very rough check. We just want to rule out system that does NOT have enough resources
 				_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
 
 				if cpuAlloc < minCoreCount {
 					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
 				}
+			})
 
-			requireSRIOVDevices()
+			// empty context to apply kubelet config changes
+			ginkgo.Context("", func() {
+				tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+					// Set the CPU Manager policy to static.
+					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
 
+					// Set the CPU Manager reconcile period to 1 second.
+					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
+
+					cpus := reservedSystemCPUs.String()
+					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
+					initialConfig.ReservedSystemCPUs = cpus
+				})
+
+				ginkgo.It("should return the expected responses", func() {
 					onlineCPUs, err := getOnlineCPUs()
 					framework.ExpectNoError(err)
 
-			// Make sure all the feature gates and the right settings are in place.
-			oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 					configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
 					sd := setupSRIOVConfigOrFail(f, configMap)
 					defer teardownSRIOVConfigOrFail(f, sd)
@@ -579,23 +585,16 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 					podresourcesListTests(f, cli, sd)
 					ginkgo.By("checking GetAllocatableResources()")
 					podresourcesGetAllocatableResourcesTests(cli, sd, onlineCPUs, reservedSystemCPUs)
-
+				})
+			})
 		})
 
-		ginkgo.It("should return the expected responses with cpumanager none policy", func() {
+		ginkgo.Context("with CPU manager None policy", func() {
+			ginkgo.It("should return the expected responses", func() {
 				// current default is "none" policy - no need to restart the kubelet
 
 				requireSRIOVDevices()
 
-			oldCfg := enablePodResourcesFeatureGateInKubelet(f)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 				configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
 				sd := setupSRIOVConfigOrFail(f, configMap)
 				defer teardownSRIOVConfigOrFail(f, sd)
@@ -615,33 +614,42 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 				// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
 				podresourcesGetAllocatableResourcesTests(cli, sd, cpuset.CPUSet{}, cpuset.CPUSet{})
 			})
-
+		})
 	})
 
-	ginkgo.Context("Without SRIOV devices in the system", func() {
+	ginkgo.Context("without SRIOV devices in the system", func() {
-		ginkgo.It("should return the expected responses with cpumanager static policy enabled", func() {
+		ginkgo.BeforeEach(func() {
+			requireLackOfSRIOVDevices()
+		})
+
+		ginkgo.Context("with CPU manager Static policy", func() {
+			ginkgo.BeforeEach(func() {
 				// this is a very rough check. We just want to rule out system that does NOT have enough resources
 				_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
 
 				if cpuAlloc < minCoreCount {
 					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
 				}
+			})
 
-			requireLackOfSRIOVDevices()
+			// empty context to apply kubelet config changes
+			ginkgo.Context("", func() {
+				tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+					// Set the CPU Manager policy to static.
+					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
 
+					// Set the CPU Manager reconcile period to 1 second.
+					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
+
+					cpus := reservedSystemCPUs.String()
+					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
+					initialConfig.ReservedSystemCPUs = cpus
+				})
+
+				ginkgo.It("should return the expected responses", func() {
 					onlineCPUs, err := getOnlineCPUs()
 					framework.ExpectNoError(err)
 
-			// Make sure all the feature gates and the right settings are in place.
-			oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 					endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
 					framework.ExpectNoError(err)
 
@@ -652,21 +660,11 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 					podresourcesListTests(f, cli, nil)
 					podresourcesGetAllocatableResourcesTests(cli, nil, onlineCPUs, reservedSystemCPUs)
 				})
+			})
+		})
 
-		ginkgo.It("should return the expected responses with cpumanager none policy", func() {
+		ginkgo.Context("with CPU manager None policy", func() {
-			// current default is "none" policy - no need to restart the kubelet
+			ginkgo.It("should return the expected responses", func() {
-
-			requireLackOfSRIOVDevices()
-
-			oldCfg := enablePodResourcesFeatureGateInKubelet(f)
-			defer func() {
-				// restore kubelet config
-				setOldKubeletConfig(oldCfg)
-
-				// Delete state file to allow repeated runs
-				deleteStateFile()
-			}()
-
 				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
 				framework.ExpectNoError(err)
 
@@ -678,10 +676,17 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 				// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
 				podresourcesGetAllocatableResourcesTests(cli, nil, cpuset.CPUSet{}, cpuset.CPUSet{})
 			})
+		})
+
+		ginkgo.Context("with disabled KubeletPodResourcesGetAllocatable feature gate", func() {
+			tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+				if initialConfig.FeatureGates == nil {
+					initialConfig.FeatureGates = make(map[string]bool)
+				}
+				initialConfig.FeatureGates[string(kubefeatures.KubeletPodResourcesGetAllocatable)] = false
+			})
 
 			ginkgo.It("should return the expected error with the feature gate disabled", func() {
-			e2eskipper.SkipIfFeatureGateEnabled(kubefeatures.KubeletPodResourcesGetAllocatable)
-
 				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
 				framework.ExpectNoError(err)
 
@@ -695,35 +700,41 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 				framework.ExpectError(err, "With feature gate disabled, the call must fail")
 			})
 		})
+	})
 
-	ginkgo.Context("Use KubeVirt device plugin, which reports resources w/o hardware topology", func() {
+	ginkgo.Context("with KubeVirt device plugin, which reports resources w/o hardware topology", func() {
-		ginkgo.It("should return proper podresources the same as before the restart of kubelet", func() {
+		ginkgo.BeforeEach(func() {
-
-			if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.KubeletPodResources) {
-				e2eskipper.Skipf("this test is meant to run with the POD Resources Extensions feature gate enabled")
-			}
-
 			_, err := os.Stat("/dev/kvm")
 			if errors.Is(err, os.ErrNotExist) {
 				e2eskipper.Skipf("KubeVirt device plugin could work only in kvm based environment")
 			}
+		})
 
+		ginkgo.Context("with CPU manager Static policy", func() {
+			ginkgo.BeforeEach(func() {
+				// this is a very rough check. We just want to rule out system that does NOT have enough resources
 				_, cpuAlloc, _ := getLocalNodeCPUDetails(f)
 
 				if cpuAlloc < minCoreCount {
 					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
 				}
+			})
 
-			// Make sure all the feature gates and the right settings are in place.
+			// empty context to apply kubelet config changes
-			oldCfg := configurePodResourcesInKubelet(f, true, reservedSystemCPUs)
+			ginkgo.Context("", func() {
-			defer func() {
+				tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-				// restore kubelet config
+					// Set the CPU Manager policy to static.
-				setOldKubeletConfig(oldCfg)
+					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
 
-				// Delete state file to allow repeated runs
+					// Set the CPU Manager reconcile period to 1 second.
-				deleteStateFile()
+					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
-			}()
 
+					cpus := reservedSystemCPUs.String()
+					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
+					initialConfig.ReservedSystemCPUs = cpus
+				})
+
+				ginkgo.It("should return proper podresources the same as before the restart of kubelet", func() {
 					dpPod := setupKubeVirtDevicePluginOrFail(f)
 					defer teardownKubeVirtDevicePluginOrFail(f, dpPod)
 
@@ -769,7 +780,8 @@ var _ = SIGDescribe("POD Resources [Serial] [Feature:PodResources][NodeFeature:P
 					expectPodResources(1, cli, []podDesc{desc})
 					tpd.deletePodsForTest(f)
 				})
-
+			})
+		})
 	})
 })
 
@@ -787,113 +799,6 @@ func getOnlineCPUs() (cpuset.CPUSet, error) {
 	return cpuset.Parse(strings.TrimSpace(string(onlineCPUList)))
 }
 
-func configurePodResourcesInKubelet(f *framework.Framework, cleanStateFile bool, reservedSystemCPUs cpuset.CPUSet) (oldCfg *kubeletconfig.KubeletConfiguration) {
-	// we also need CPUManager with static policy to be able to do meaningful testing
-	oldCfg, err := getCurrentKubeletConfig()
-	framework.ExpectNoError(err)
-	newCfg := oldCfg.DeepCopy()
-	newCfg.FeatureGates[string(kubefeatures.KubeletPodResourcesGetAllocatable)] = true
-
-	// After graduation of the CPU Manager feature to Beta, the CPU Manager
-	// "none" policy is ON by default. But when we set the CPU Manager policy to
-	// "static" in this test and the Kubelet is restarted so that "static"
-	// policy can take effect, there will always be a conflict with the state
-	// checkpointed in the disk (i.e., the policy checkpointed in the disk will
-	// be "none" whereas we are trying to restart Kubelet with "static"
-	// policy). Therefore, we delete the state file so that we can proceed
-	// with the tests.
-	// Only delete the state file at the begin of the tests.
-	if cleanStateFile {
-		deleteStateFile()
-	}
-
-	// Set the CPU Manager policy to static.
-	newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
-
-	// Set the CPU Manager reconcile period to 1 second.
-	newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
-
-	if reservedSystemCPUs.Size() > 0 {
-		cpus := reservedSystemCPUs.String()
-		framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
-		newCfg.ReservedSystemCPUs = cpus
-	} else {
-		// The Kubelet panics if either kube-reserved or system-reserved is not set
-		// when CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
-		// kubelet doesn't panic.
-		if newCfg.KubeReserved == nil {
-			newCfg.KubeReserved = map[string]string{}
-		}
-
-		if _, ok := newCfg.KubeReserved["cpu"]; !ok {
-			newCfg.KubeReserved["cpu"] = "200m"
-		}
-	}
-	// Update the Kubelet configuration.
-	ginkgo.By("Stopping the kubelet")
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// wait until the kubelet health check will succeed
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
-
-	// Wait for the Kubelet to be ready.
-	gomega.Eventually(func() bool {
-		nodes, err := e2enode.TotalReady(f.ClientSet)
-		framework.ExpectNoError(err)
-		return nodes == 1
-	}, time.Minute, time.Second).Should(gomega.BeTrue())
-
-	return oldCfg
-}
-
-func enablePodResourcesFeatureGateInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.KubeletConfiguration) {
-	oldCfg, err := getCurrentKubeletConfig()
-	framework.ExpectNoError(err)
-	newCfg := oldCfg.DeepCopy()
-	if newCfg.FeatureGates == nil {
-		newCfg.FeatureGates = make(map[string]bool)
-	}
-
-	ginkgo.By("Stopping the kubelet")
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// wait until the kubelet health check will succeed
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
-
-	// Wait for the Kubelet to be ready.
-	gomega.Eventually(func() bool {
-		nodes, err := e2enode.TotalReady(f.ClientSet)
-		framework.ExpectNoError(err)
-		return nodes == 1
-	}, time.Minute, time.Second).Should(gomega.BeTrue())
-
-	return oldCfg
-}
-
 func setupKubeVirtDevicePluginOrFail(f *framework.Framework) *v1.Pod {
 	e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)
 

test/e2e_node/topology_manager_test.go

@@ -42,7 +42,6 @@ import (
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
-	e2enodekubelet "k8s.io/kubernetes/test/e2e_node/kubeletconfig"
 	testutils "k8s.io/kubernetes/test/utils"
 
 	"github.com/onsi/ginkgo"
@@ -194,7 +193,7 @@ func findNUMANodeWithoutSRIOVDevices(configMap *v1.ConfigMap, numaNodes int) (in
 	return findNUMANodeWithoutSRIOVDevicesFromSysfs(numaNodes)
 }
 
-func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletconfig.KubeletConfiguration, policy, scope string, configMap *v1.ConfigMap, numaNodes int) string {
+func configureTopologyManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, policy, scope string, configMap *v1.ConfigMap, numaNodes int) (*kubeletconfig.KubeletConfiguration, string) {
 	// Configure Topology Manager in Kubelet with policy.
 	newCfg := oldCfg.DeepCopy()
 	if newCfg.FeatureGates == nil {
@@ -204,8 +203,6 @@ func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletco
 	newCfg.FeatureGates["CPUManager"] = true
 	newCfg.FeatureGates["TopologyManager"] = true
 
-	deleteStateFile()
-
 	// Set the Topology Manager policy
 	newCfg.TopologyManagerPolicy = policy
 
@@ -237,27 +234,7 @@ func configureTopologyManagerInKubelet(f *framework.Framework, oldCfg *kubeletco
 	// Dump the config -- debug
 	framework.Logf("New kubelet config is %s", *newCfg)
 
-	ginkgo.By("Stopping the kubelet")
+	return newCfg, newCfg.ReservedSystemCPUs
-	startKubelet := stopKubelet()
-
-	// wait until the kubelet health check will fail
-	gomega.Eventually(func() bool {
-		return kubeletHealthCheck(kubeletHealthCheckURL)
-	}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
-
-	ginkgo.By("Starting the kubelet")
-	startKubelet()
-
-	// Wait for the Kubelet to be ready.
-	gomega.Eventually(func() bool {
-		nodes, err := e2enode.TotalReady(f.ClientSet)
-		framework.ExpectNoError(err)
-		return nodes == 1
-	}, time.Minute, time.Second).Should(gomega.BeTrue())
-
-	return newCfg.ReservedSystemCPUs
 }
 
 // getSRIOVDevicePluginPod returns the Device Plugin pod for sriov resources in e2e tests.
@@ -910,7 +887,8 @@ func runTopologyManagerTests(f *framework.Framework) {
 			ginkgo.By(fmt.Sprintf("by configuring Topology Manager policy to %s", policy))
 			framework.Logf("Configuring topology Manager policy to %s", policy)
 
-			configureTopologyManagerInKubelet(f, oldCfg, policy, scope, nil, 0)
+			newCfg, _ := configureTopologyManagerInKubelet(oldCfg, policy, scope, nil, 0)
+			updateKubeletConfig(f, newCfg, true)
 			// Run the tests
 			runTopologyManagerPolicySuiteTests(f)
 		}
@@ -933,7 +911,8 @@ func runTopologyManagerTests(f *framework.Framework) {
 			ginkgo.By(fmt.Sprintf("by configuring Topology Manager policy to %s", policy))
 			framework.Logf("Configuring topology Manager policy to %s", policy)
 
-			reservedSystemCPUs := configureTopologyManagerInKubelet(f, oldCfg, policy, scope, configMap, numaNodes)
+			newCfg, reservedSystemCPUs := configureTopologyManagerInKubelet(oldCfg, policy, scope, configMap, numaNodes)
+			updateKubeletConfig(f, newCfg, true)
 
 			runTopologyManagerNodeAlignmentSuiteTests(f, sd, reservedSystemCPUs, policy, numaNodes, coreCount)
 		}
@@ -950,14 +929,15 @@ func runTopologyManagerTests(f *framework.Framework) {
 		policy := topologymanager.PolicySingleNumaNode
 		scope := podScopeTopology
 
-		reservedSystemCPUs := configureTopologyManagerInKubelet(f, oldCfg, policy, scope, configMap, numaNodes)
+		newCfg, reservedSystemCPUs := configureTopologyManagerInKubelet(oldCfg, policy, scope, configMap, numaNodes)
+		updateKubeletConfig(f, newCfg, true)
 
 		runTMScopeResourceAlignmentTestSuite(f, configMap, reservedSystemCPUs, policy, numaNodes, coreCount)
 	})
 
 	ginkgo.AfterEach(func() {
 		// restore kubelet config
-		setOldKubeletConfig(oldCfg)
+		updateKubeletConfig(f, oldCfg, true)
 	})
 }
 

test/e2e_node/util.go

@@ -75,6 +75,9 @@ const (
 	defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
 	kubeletReadOnlyPort        = "10255"
 	kubeletHealthCheckURL      = "http://127.0.0.1:" + kubeletReadOnlyPort + "/healthz"
+	// state files
+	cpuManagerStateFile    = "/var/lib/kubelet/cpu_manager_state"
+	memoryManagerStateFile = "/var/lib/kubelet/memory_manager_state"
 )
 
 func getNodeSummary() (*stats.Summary, error) {
@@ -158,36 +161,29 @@ func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) {
 // Returns true on success.
 func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(initialConfig *kubeletconfig.KubeletConfiguration)) {
 	var oldCfg *kubeletconfig.KubeletConfiguration
+
 	ginkgo.BeforeEach(func() {
 		oldCfg, err := getCurrentKubeletConfig()
 		framework.ExpectNoError(err)
+
 		newCfg := oldCfg.DeepCopy()
 		updateFunction(newCfg)
 		if apiequality.Semantic.DeepEqual(*newCfg, *oldCfg) {
 			return
 		}
 
-		// Update the Kubelet configuration.
+		updateKubeletConfig(f, newCfg, true)
-		ginkgo.By("Stopping the kubelet")
-		startKubelet := stopKubelet()
-
-		// wait until the kubelet health check will fail
-		gomega.Eventually(func() bool {
-			return kubeletHealthCheck(kubeletHealthCheckURL)
-		}, time.Minute, time.Second).Should(gomega.BeFalse())
-
-		framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(newCfg))
-
-		ginkgo.By("Starting the kubelet")
-		startKubelet()
-
-		// wait until the kubelet health check will succeed
-		gomega.Eventually(func() bool {
-			return kubeletHealthCheck(kubeletHealthCheckURL)
-		}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
 	})
+
 	ginkgo.AfterEach(func() {
 		if oldCfg != nil {
+			// Update the Kubelet configuration.
+			updateKubeletConfig(f, oldCfg, true)
+		}
+	})
+}
+
+func updateKubeletConfig(f *framework.Framework, kubeletConfig *kubeletconfig.KubeletConfiguration, deleteStateFiles bool) {
 	// Update the Kubelet configuration.
 	ginkgo.By("Stopping the kubelet")
 	startKubelet := stopKubelet()
@@ -197,7 +193,13 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini
 		return kubeletHealthCheck(kubeletHealthCheckURL)
 	}, time.Minute, time.Second).Should(gomega.BeFalse())
 
-			framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(oldCfg))
+	// Delete CPU and memory manager state files to be sure it will not prevent the kubelet restart
+	if deleteStateFiles {
+		deleteStateFile(cpuManagerStateFile)
+		deleteStateFile(memoryManagerStateFile)
+	}
+
+	framework.ExpectNoError(e2enodekubelet.WriteKubeletConfigFile(kubeletConfig))
 
 	ginkgo.By("Starting the kubelet")
 	startKubelet()
@@ -206,8 +208,18 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini
 	gomega.Eventually(func() bool {
 		return kubeletHealthCheck(kubeletHealthCheckURL)
 	}, 2*time.Minute, 5*time.Second).Should(gomega.BeTrue())
+
+	// Wait for the Kubelet to be ready.
+	gomega.Eventually(func() bool {
+		nodes, err := e2enode.TotalReady(f.ClientSet)
+		framework.ExpectNoError(err)
+		return nodes == 1
+	}, time.Minute, time.Second).Should(gomega.BeTrue())
 }
-	})
+
+func deleteStateFile(stateFileName string) {
+	err := exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", stateFileName)).Run()
+	framework.ExpectNoError(err, "failed to delete the state file")
 }
 
 // Returns true if kubeletConfig is enabled, false otherwise or if we cannot determine if it is.