From 90906ab1ea8d4331dec49db2b5e452d3dba8701d Mon Sep 17 00:00:00 2001 From: Luca Berneking Date: Thu, 1 Aug 2024 14:19:27 +0200 Subject: [PATCH] Reduce memory usage/allocations during wait for volume attachment After a node restart kubelet tries to (re)attach all volumes to the pods. We poll the `verifyVolumesMountedFunc` every 300ms to check whether the mount has succeeded. This function called the `GetMountedVolumesForPod` function that allocates memory for every volumes on every pod (`len(asw.attachedVolumes)`). Because this function is executed for every pod simultaneously, this results in exponential memory usage and high cpu usage due to garbage collection. We already know the exact volume names and pod name and are able to completly remove the slice allocation. Signed-off-by: Luca Berneking --- .../cache/actual_state_of_world.go | 38 ++++++++++++++++++- pkg/kubelet/volumemanager/volume_manager.go | 10 ++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 38017a588f7..8ef518ee56f 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -120,6 +120,9 @@ type ActualStateOfWorld interface { // and false is returned. PodRemovedFromVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName) bool + // PodHasMountedVolumes returns true if any volume is mounted on the given pod + PodHasMountedVolumes(podName volumetypes.UniquePodName) bool + // VolumeExistsWithSpecName returns true if the given volume specified with the // volume spec name (a.k.a., InnerVolumeSpecName) exists in the list of // volumes that should be attached to this node. @@ -146,6 +149,10 @@ type ActualStateOfWorld interface { // current actual state of the world. GetMountedVolumesForPod(podName volumetypes.UniquePodName) []MountedVolume + // GetMountedVolumeForPodByOuterVolumeSpecName returns the volume and true if + // the given outerVolumeSpecName is mounted on the given pod. + GetMountedVolumeForPodByOuterVolumeSpecName(podName volumetypes.UniquePodName, outerVolumeSpecName string) (MountedVolume, bool) + // GetPossiblyMountedVolumesForPod generates and returns a list of volumes for // the specified pod that either are attached and mounted or are "uncertain", // i.e. a volume plugin may be mounting the volume right now. @@ -948,6 +955,20 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa return podExists, volumeObj.devicePath, nil } +func (asw *actualStateOfWorld) PodHasMountedVolumes(podName volumetypes.UniquePodName) bool { + asw.RLock() + defer asw.RUnlock() + for _, volumeObj := range asw.attachedVolumes { + if podObj, hasPod := volumeObj.mountedPods[podName]; hasPod { + if podObj.volumeMountStateForPod == operationexecutor.VolumeMounted { + return true + } + } + } + + return false +} + func (asw *actualStateOfWorld) volumeNeedsExpansion(volumeObj attachedVolume, desiredVolumeSize resource.Quantity) (resource.Quantity, bool) { currentSize := resource.Quantity{} if volumeObj.persistentVolumeSize != nil { @@ -1061,7 +1082,7 @@ func (asw *actualStateOfWorld) GetMountedVolumesForPod( podName volumetypes.UniquePodName) []MountedVolume { asw.RLock() defer asw.RUnlock() - mountedVolume := make([]MountedVolume, 0 /* len */, len(asw.attachedVolumes) /* cap */) + mountedVolume := make([]MountedVolume, 0 /* len */) for _, volumeObj := range asw.attachedVolumes { for mountedPodName, podObj := range volumeObj.mountedPods { if mountedPodName == podName && podObj.volumeMountStateForPod == operationexecutor.VolumeMounted { @@ -1075,6 +1096,21 @@ func (asw *actualStateOfWorld) GetMountedVolumesForPod( return mountedVolume } +func (asw *actualStateOfWorld) GetMountedVolumeForPodByOuterVolumeSpecName( + podName volumetypes.UniquePodName, outerVolumeSpecName string) (MountedVolume, bool) { + asw.RLock() + defer asw.RUnlock() + for _, volumeObj := range asw.attachedVolumes { + if podObj, hasPod := volumeObj.mountedPods[podName]; hasPod { + if podObj.volumeMountStateForPod == operationexecutor.VolumeMounted && podObj.outerVolumeSpecName == outerVolumeSpecName { + return getMountedVolume(&podObj, &volumeObj), true + } + } + } + + return MountedVolume{}, false +} + func (asw *actualStateOfWorld) GetPossiblyMountedVolumesForPod( podName volumetypes.UniquePodName) []MountedVolume { asw.RLock() diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 79bb91c2f3a..37527d99f20 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -514,7 +514,13 @@ func (vm *volumeManager) verifyVolumesMountedFunc(podName types.UniquePodName, e if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 { return true, errors.New(strings.Join(errs, "; ")) } - return len(vm.getUnmountedVolumes(podName, expectedVolumes)) == 0, nil + for _, expectedVolume := range expectedVolumes { + _, found := vm.actualStateOfWorld.GetMountedVolumeForPodByOuterVolumeSpecName(podName, expectedVolume) + if !found { + return false, nil + } + } + return true, nil } } @@ -525,7 +531,7 @@ func (vm *volumeManager) verifyVolumesUnmountedFunc(podName types.UniquePodName) if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 { return true, errors.New(strings.Join(errs, "; ")) } - return len(vm.actualStateOfWorld.GetMountedVolumesForPod(podName)) == 0, nil + return !vm.actualStateOfWorld.PodHasMountedVolumes(podName), nil } }