diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 6bdb7715eec..cdfd24923ff 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -144,7 +144,8 @@ func NewAttachDetachController( adc.desiredStateOfWorld, adc.actualStateOfWorld, adc.attacherDetacher, - adc.nodeStatusUpdater) + adc.nodeStatusUpdater, + recorder) adc.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator( desiredStateOfWorldPopulatorLoopSleepPeriod, diff --git a/pkg/controller/volume/attachdetach/cache/desired_state_of_world.go b/pkg/controller/volume/attachdetach/cache/desired_state_of_world.go index aa4371773f7..93352bd4565 100644 --- a/pkg/controller/volume/attachdetach/cache/desired_state_of_world.go +++ b/pkg/controller/volume/attachdetach/cache/desired_state_of_world.go @@ -158,6 +158,10 @@ type nodeManaged struct { // The volume object represents a volume that should be attached to a node. type volumeToAttach struct { + // multiAttachErrorReported indicates whether the multi-attach error has been reported for the given volume. + // It is used to to prevent reporting the error from being reported more than once for a given volume. + multiAttachErrorReported bool + // volumeName contains the unique identifier for this volume. volumeName v1.UniqueVolumeName @@ -231,9 +235,10 @@ func (dsw *desiredStateOfWorld) AddPod( volumeObj, volumeExists := nodeObj.volumesToAttach[volumeName] if !volumeExists { volumeObj = volumeToAttach{ - volumeName: volumeName, - spec: volumeSpec, - scheduledPods: make(map[types.UniquePodName]pod), + multiAttachErrorReported: false, + volumeName: volumeName, + spec: volumeSpec, + scheduledPods: make(map[types.UniquePodName]pod), } dsw.nodesManaged[nodeName].volumesToAttach[volumeName] = volumeObj } @@ -349,10 +354,11 @@ func (dsw *desiredStateOfWorld) GetVolumesToAttach() []VolumeToAttach { volumesToAttach = append(volumesToAttach, VolumeToAttach{ VolumeToAttach: operationexecutor.VolumeToAttach{ - VolumeName: volumeName, - VolumeSpec: volumeObj.spec, - NodeName: nodeName, - ScheduledPods: getPodsFromMap(volumeObj.scheduledPods), + MultiAttachErrorReported: volumeObj.multiAttachErrorReported, + VolumeName: volumeName, + VolumeSpec: volumeObj.spec, + NodeName: nodeName, + ScheduledPods: getPodsFromMap(volumeObj.scheduledPods), }}) } } diff --git a/pkg/controller/volume/attachdetach/reconciler/BUILD b/pkg/controller/volume/attachdetach/reconciler/BUILD index e537530f305..3ff4f931fd6 100644 --- a/pkg/controller/volume/attachdetach/reconciler/BUILD +++ b/pkg/controller/volume/attachdetach/reconciler/BUILD @@ -16,11 +16,13 @@ go_library( "//pkg/api/v1:go_default_library", "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/statusupdater:go_default_library", + "//pkg/kubelet/events:go_default_library", "//pkg/util/goroutinemap/exponentialbackoff:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler.go b/pkg/controller/volume/attachdetach/reconciler/reconciler.go index ae0a895387f..3b56bfcfa1b 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler.go @@ -25,9 +25,11 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater" + kevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -63,7 +65,8 @@ func NewReconciler( desiredStateOfWorld cache.DesiredStateOfWorld, actualStateOfWorld cache.ActualStateOfWorld, attacherDetacher operationexecutor.OperationExecutor, - nodeStatusUpdater statusupdater.NodeStatusUpdater) Reconciler { + nodeStatusUpdater statusupdater.NodeStatusUpdater, + recorder record.EventRecorder) Reconciler { return &reconciler{ loopPeriod: loopPeriod, maxWaitForUnmountDuration: maxWaitForUnmountDuration, @@ -74,6 +77,7 @@ func NewReconciler( attacherDetacher: attacherDetacher, nodeStatusUpdater: nodeStatusUpdater, timeOfLastSync: time.Now(), + recorder: recorder, } } @@ -87,6 +91,7 @@ type reconciler struct { nodeStatusUpdater statusupdater.NodeStatusUpdater timeOfLastSync time.Time disableReconciliationSync bool + recorder record.EventRecorder } func (rc *reconciler) Run(stopCh <-chan struct{}) { @@ -248,7 +253,14 @@ func (rc *reconciler) reconcile() { if rc.isMultiAttachForbidden(volumeToAttach.VolumeSpec) { nodes := rc.actualStateOfWorld.GetNodesForVolume(volumeToAttach.VolumeName) if len(nodes) > 0 { - glog.V(4).Infof("Volume %q is already exclusively attached to node %q and can't be attached to %q", volumeToAttach.VolumeName, nodes, volumeToAttach.NodeName) + if !volumeToAttach.MultiAttachErrorReported { + simpleMsg, detailedMsg := volumeToAttach.GenerateMsg("Multi-Attach error", "Volume is already exclusively attached to one node and can't be attached to another") + for _, pod := range volumeToAttach.ScheduledPods { + rc.recorder.Eventf(pod, v1.EventTypeWarning, kevents.FailedAttachVolume, simpleMsg) + } + volumeToAttach.MultiAttachErrorReported = true + glog.Warningf(detailedMsg) + } continue } } diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go index c1118898376..ec011918abb 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go @@ -55,7 +55,7 @@ func Test_Run_Positive_DoNothing(t *testing.T) { nsu := statusupdater.NewNodeStatusUpdater( fakeKubeClient, informerFactory.Core().V1().Nodes().Lister(), asw) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) // Act ch := make(chan struct{}) @@ -83,7 +83,7 @@ func Test_Run_Positive_OneDesiredVolumeAttach(t *testing.T) { ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName := "pod-uid" volumeName := v1.UniqueVolumeName("volume-name") volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) @@ -129,7 +129,7 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithUnmountedVolume(t *te ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName := "pod-uid" volumeName := v1.UniqueVolumeName("volume-name") volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) @@ -196,7 +196,7 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithMountedVolume(t *test ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName := "pod-uid" volumeName := v1.UniqueVolumeName("volume-name") volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) @@ -263,7 +263,7 @@ func Test_Run_Negative_OneDesiredVolumeAttachThenDetachWithUnmountedVolumeUpdate ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(true /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName := "pod-uid" volumeName := v1.UniqueVolumeName("volume-name") volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) @@ -333,7 +333,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing. ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName1 := "pod-uid1" podName2 := "pod-uid2" volumeName := v1.UniqueVolumeName("volume-name") @@ -423,7 +423,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing. ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */)) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) reconciler := NewReconciler( - reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu) + reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder) podName1 := "pod-uid1" podName2 := "pod-uid2" volumeName := v1.UniqueVolumeName("volume-name") diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index d7ef72b5cab..0a0efc20cfd 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -47,6 +47,7 @@ const ( NodeNotSchedulable = "NodeNotSchedulable" StartingKubelet = "Starting" KubeletSetupFailed = "KubeletSetupFailed" + FailedAttachVolume = "FailedAttachVolume" FailedDetachVolume = "FailedDetachVolume" FailedMountVolume = "FailedMount" FailedUnMountVolume = "FailedUnMount" diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 3225cacd44d..ff914bf19f6 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -211,6 +211,10 @@ func generateVolumeMsg(prefixMsg, suffixMsg, volumeName, details string) (simple // VolumeToAttach represents a volume that should be attached to a node. type VolumeToAttach struct { + // MultiAttachErrorReported indicates whether the multi-attach error has been reported for the given volume. + // It is used to to prevent reporting the error from being reported more than once for a given volume. + MultiAttachErrorReported bool + // VolumeName is the unique identifier for the volume that should be // attached. VolumeName v1.UniqueVolumeName