Merge pull request #137708 from tchap/kcm-add-leaderelection-flag

kcm: Link the leader election release on exit feature gate

cmd/kube-controller-manager/app/controllermanager.go

@@ -76,6 +76,7 @@ import (
 	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/controller-manager/controller"
 	"k8s.io/controller-manager/pkg/clientbuilder"
+	cmfeatures "k8s.io/controller-manager/pkg/features"
 	controllerhealthz "k8s.io/controller-manager/pkg/healthz"
 	"k8s.io/controller-manager/pkg/informerfactory"
 	"k8s.io/controller-manager/pkg/leadermigration"
@@ -154,7 +155,11 @@ controller, and serviceaccounts controller.`,
 			fg.(featuregate.MutableFeatureGate).AddMetrics()
 			// add component version metrics
 			s.ComponentGlobalsRegistry.AddMetrics()
-			return Run(server.SetupSignalContext(), c.Complete())
+
+			if utilfeature.DefaultFeatureGate.Enabled(cmfeatures.ControllerManagerReleaseLeaderElectionLockOnExit) {
+				ctx = server.SetupSignalContext()
+			}
+			return Run(ctx, c.Complete())
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
 			for _, arg := range args {
@@ -406,6 +411,9 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
 				}),
 				OnStoppedLeading: func() {
 					logger.Error(nil, "leaderelection lost/stopped")
+					if !utilfeature.DefaultFeatureGate.Enabled(cmfeatures.ControllerManagerReleaseLeaderElectionLockOnExit) {
+						klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+					}
 				},
 			})
 	})
@@ -434,6 +442,9 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
 						}),
 						OnStoppedLeading: func() {
 							logger.Error(nil, "migration leaderelection lost/stopped")
+							if !utilfeature.DefaultFeatureGate.Enabled(cmfeatures.ControllerManagerReleaseLeaderElectionLockOnExit) {
+								klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+							}
 						},
 					})
 			})
@@ -843,14 +854,15 @@ func leaderElectAndRun(ctx context.Context, c *config.CompletedConfig, lockIdent
 	}
 
 	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock:          rl,
-		LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration,
-		RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration,
-		RetryPeriod:   c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration,
-		Callbacks:     callbacks,
-		WatchDog:      electionChecker,
-		Name:          leaseName,
-		Coordinated:   utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection),
+		Lock:            rl,
+		LeaseDuration:   c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration,
+		RenewDeadline:   c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration,
+		RetryPeriod:     c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration,
+		Callbacks:       callbacks,
+		WatchDog:        electionChecker,
+		ReleaseOnCancel: utilfeature.DefaultFeatureGate.Enabled(cmfeatures.ControllerManagerReleaseLeaderElectionLockOnExit),
+		Name:            leaseName,
+		Coordinated:     utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection),
 	})
 }
 

test/integration/controllermanager/leader_election_release_test.go

@@ -0,0 +1,110 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllermanager
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	coordinationv1 "k8s.io/api/coordination/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
+	kubecontrollermanagertesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	"k8s.io/kubernetes/test/integration/framework"
+	"k8s.io/kubernetes/test/utils/kubeconfig"
+)
+
+// TestLeaderElectionReleaseOnCancel verifies that when the
+// ControllerManagerReleaseLeaderElectionLockOnExit feature gate is enabled,
+// tearing down the KCM (cancelling its context) causes the leader election
+// lease to be released (holder identity cleared). This allows a new leader
+// to acquire the lock without waiting for the full lease duration to expire.
+//
+// Note: We only test the feature-enabled case here. The feature-disabled case
+// retains the legacy behavior where OnStoppedLeading calls klog.FlushAndExit
+// (os.Exit), which cannot be tested in-process.
+func TestLeaderElectionReleaseOnCancel(t *testing.T) {
+	// Start a real API server.
+	apiServer := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
+	t.Cleanup(apiServer.TearDownFn)
+
+	client, err := kubernetes.NewForConfig(apiServer.ClientConfig)
+	if err != nil {
+		t.Fatalf("failed to create client: %v", err)
+	}
+
+	// Write a kubeconfig file for the KCM to use.
+	clientConfig := kubeconfig.CreateKubeConfig(apiServer.ClientConfig)
+	kubeConfigFile := filepath.Join(t.TempDir(), "kubeconfig.yaml")
+	if err := clientcmd.WriteToFile(*clientConfig, kubeConfigFile); err != nil {
+		t.Fatalf("failed to write kubeconfig: %v", err)
+	}
+
+	// Start a full KCM with leader election and the feature gate enabled.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	kcm, err := kubecontrollermanagertesting.StartTestServer(t, ctx, []string{
+		"--kubeconfig=" + kubeConfigFile,
+		"--leader-elect=true",
+		"--feature-gates=ControllerManagerReleaseLeaderElectionLockOnExit=true",
+	})
+	if err != nil {
+		t.Fatalf("failed to start KCM: %v", err)
+	}
+	t.Cleanup(kcm.TearDownFn)
+
+	// Wait for the lease to exist and have a holder. There can be a brief gap
+	// between the KCM's /healthz becoming ready and the lease being acquired.
+	var lease *coordinationv1.Lease
+	err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) {
+		lease, err = client.CoordinationV1().Leases("kube-system").Get(ctx, "kube-controller-manager", metav1.GetOptions{})
+		if err != nil {
+			return false, nil // retry on not-found or transient errors
+		}
+		return lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity != "", nil
+	})
+	if err != nil {
+		t.Fatalf("timed out waiting for lease to have a holder: %v", err)
+	}
+	t.Logf("lease holder before shutdown: %q", *lease.Spec.HolderIdentity)
+
+	// Tear down the KCM (cancels its context), which should release the lease.
+	kcm.TearDownFn()
+
+	// Poll until the lease holder is cleared.
+	err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) {
+		lease, err = client.CoordinationV1().Leases("kube-system").Get(ctx, "kube-controller-manager", metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "", nil
+	})
+	if err != nil {
+		holder := "<nil>"
+		if lease.Spec.HolderIdentity != nil {
+			holder = *lease.Spec.HolderIdentity
+		}
+		t.Fatalf("expected lease holder to be cleared after shutdown, but got %q: %v", holder, err)
+	}
+	t.Log("lease holder cleared after shutdown")
+}