From a910d830706323be30066ca8df2af7014079d81d Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 26 Oct 2022 14:05:23 +0200 Subject: [PATCH 1/2] Reduce log noise on SELinux mount mismatch The Desired State of World can require a different SELinux mount context than is in the Actual State of World and it's perfectly OK. For example when user changes SELinux context of Pods or when the context is reconstructed after kubelet restart. Don't spam log and don't report errors to the user as event - reconciler will do the right thing and unmount the old volume (with wrong context) and mount a new one in the next reconciliation. It's not an error, it's expected workflow. --- pkg/kubelet/volumemanager/reconciler/reconciler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 35c14a54c8e..14b4a660c46 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -220,10 +220,10 @@ func (rc *reconciler) mountOrAttachVolumes() { volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, volumeToMount.PersistentVolumeSize, volumeToMount.SELinuxLabel) volumeToMount.DevicePath = devicePath if cache.IsSELinuxMountMismatchError(err) { - // TODO: check error message + lower frequency, this can be noisy - klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("mount precondition failed, please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file", err).Error(), "pod", klog.KObj(volumeToMount.Pod)) - // TODO: report error better, this may be too noisy - rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error()) + // The volume is mounted, but with an unexpected SELinux context. + // It will get unmounted in unmountVolumes / unmountDetachDevices and + // then removed from actualStateOfWorld. + continue } else if cache.IsVolumeNotAttachedError(err) { rc.waitForVolumeAttach(volumeToMount) } else if !volMounted || cache.IsRemountRequiredError(err) { From d37808faaed7135f4a13a4ed745952ebdcda5845 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 31 Oct 2022 13:57:18 +0100 Subject: [PATCH 2/2] Report error on a pod startup on SELinux mismatch When a volume is already mounted with an unexpected SELinux label, kubelet must unmount it first and then mount it back with the expected one. Report an error to user, just in case the unmount takes too long. In therory, this error should not happen too often, because two Pods with different SELinux label will not enter Desired State of World, see dsw.AddPodToVolume. It can happen when DSW and ASW SELinux labels only when a volume has been deleted from DSW (= Pod was deleted) or a volume was reconstructed after kubelet restart. In both cases, volume manager should unmount the volume quickly. --- pkg/kubelet/volumemanager/cache/actual_state_of_world.go | 2 +- pkg/kubelet/volumemanager/reconciler/reconciler.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index e23dc4dad7d..831324813de 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -1185,7 +1185,7 @@ type seLinuxMountMismatchError struct { func (err seLinuxMountMismatchError) Error() string { return fmt.Sprintf( - "volumeName %q is already mounted to a different pod with a different SELinux label", + "waiting for unmount of volume %q, because it is already mounted to a different pod with a different SELinux label", err.volumeName) } diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 14b4a660c46..de178350158 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -223,6 +223,7 @@ func (rc *reconciler) mountOrAttachVolumes() { // The volume is mounted, but with an unexpected SELinux context. // It will get unmounted in unmountVolumes / unmountDetachDevices and // then removed from actualStateOfWorld. + rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error()) continue } else if cache.IsVolumeNotAttachedError(err) { rc.waitForVolumeAttach(volumeToMount)