Merge pull request #113307 from andrewsykim/apiserver-identity-hostname

apiserver identity: use persistent names for lease objects

pkg/controlplane/instance.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"reflect"
 	"strconv"
 	"time"
@@ -60,6 +61,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	apiserverfeatures "k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/generic"
@@ -470,13 +472,18 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 			if err != nil {
 				return err
 			}
+
+			leaseName := m.GenericAPIServer.APIServerID
+			holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID())
+
 			controller := lease.NewController(
 				clock.RealClock{},
 				kubeClient,
-				m.GenericAPIServer.APIServerID,
+				holderIdentity,
 				int32(c.ExtraConfig.IdentityLeaseDurationSeconds),
 				nil,
 				time.Duration(c.ExtraConfig.IdentityLeaseRenewIntervalSeconds)*time.Second,
+				leaseName,
 				metav1.NamespaceSystem,
 				labelAPIServerHeartbeat)
 			go controller.Run(hookContext.StopCh)
@@ -515,6 +522,14 @@ func labelAPIServerHeartbeat(lease *coordinationapiv1.Lease) error {
 	}
 	// This label indicates that kube-apiserver owns this identity lease object
 	lease.Labels[IdentityLeaseComponentLabelKey] = KubeAPIServer
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		return err
+	}
+
+	// convenience label to easily map a lease object to a specific apiserver
+	lease.Labels[apiv1.LabelHostname] = hostname
 	return nil
 }
 

pkg/kubelet/kubelet.go

@@ -835,6 +835,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		kubeCfg.NodeLeaseDurationSeconds,
 		klet.onRepeatedHeartbeatFailure,
 		renewInterval,
+		string(klet.nodeName),
 		v1.NamespaceNodeLease,
 		util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName)))
 

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -19,8 +19,10 @@ package server
 import (
 	"context"
 	"fmt"
+	"hash/fnv"
 	"net"
 	"net/http"
+	"os"
 	goruntime "runtime"
 	"runtime/debug"
 	"sort"
@@ -328,7 +330,14 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 	defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
 	var id string
 	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
-		id = "kube-apiserver-" + uuid.New().String()
+		hostname, err := os.Hostname()
+		if err != nil {
+			klog.Fatalf("error getting hostname for apiserver identity: %v", err)
+		}
+
+		h := fnv.New32a()
+		h.Write([]byte(hostname))
+		id = "kube-apiserver-" + fmt.Sprint(h.Sum32())
 	}
 	lifecycleSignals := newLifecycleSignals()
 

staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go

@@ -54,6 +54,7 @@ type controller struct {
 	client                     clientset.Interface
 	leaseClient                coordclientset.LeaseInterface
 	holderIdentity             string
+	leaseName                  string
 	leaseNamespace             string
 	leaseDurationSeconds       int32
 	renewInterval              time.Duration
@@ -71,7 +72,7 @@ type controller struct {
 }
 
 // NewController constructs and returns a controller
-func NewController(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller {
+func NewController(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseName, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller {
 	var leaseClient coordclientset.LeaseInterface
 	if client != nil {
 		leaseClient = client.CoordinationV1().Leases(leaseNamespace)
@@ -80,6 +81,7 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity
 		client:                     client,
 		leaseClient:                leaseClient,
 		holderIdentity:             holderIdentity,
+		leaseName:                  leaseName,
 		leaseNamespace:             leaseNamespace,
 		leaseDurationSeconds:       leaseDurationSeconds,
 		renewInterval:              renewInterval,
@@ -151,7 +153,7 @@ func (c *controller) backoffEnsureLease() (*coordinationv1.Lease, bool) {
 // ensureLease creates the lease if it does not exist. Returns the lease and
 // a bool (true if this call created the lease), or any error that occurs.
 func (c *controller) ensureLease() (*coordinationv1.Lease, bool, error) {
-	lease, err := c.leaseClient.Get(context.TODO(), c.holderIdentity, metav1.GetOptions{})
+	lease, err := c.leaseClient.Get(context.TODO(), c.leaseName, metav1.GetOptions{})
 	if apierrors.IsNotFound(err) {
 		// lease does not exist, create it.
 		leaseToCreate, err := c.newLease(nil)
@@ -208,7 +210,7 @@ func (c *controller) newLease(base *coordinationv1.Lease) (*coordinationv1.Lease
 	if base == nil {
 		lease = &coordinationv1.Lease{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      c.holderIdentity,
+				Name:      c.leaseName,
 				Namespace: c.leaseNamespace,
 			},
 			Spec: coordinationv1.LeaseSpec{

staging/src/k8s.io/component-helpers/apimachinery/lease/controller_test.go

@@ -59,6 +59,7 @@ func TestNewNodeLease(t *testing.T) {
 			desc: "nil base without node",
 			controller: &controller{
 				client:               fake.NewSimpleClientset(),
+				leaseName:            node.Name,
 				holderIdentity:       node.Name,
 				leaseDurationSeconds: 10,
 				clock:                fakeClock,
@@ -80,6 +81,7 @@ func TestNewNodeLease(t *testing.T) {
 			desc: "nil base with node",
 			controller: &controller{
 				client:               fake.NewSimpleClientset(node),
+				leaseName:            node.Name,
 				holderIdentity:       node.Name,
 				leaseDurationSeconds: 10,
 				clock:                fakeClock,

test/integration/controlplane/apiserver_identity_test.go

@@ -18,11 +18,14 @@ package controlplane
 
 import (
 	"context"
-	"strings"
+	"fmt"
+	"hash/fnv"
+	"os"
 	"testing"
 	"time"
 
 	coordinationv1 "k8s.io/api/coordination/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -40,6 +43,12 @@ const (
 	testLeaseName = "apiserver-lease-test"
 )
 
+func expectedAPIServerIdentity(hostname string) string {
+	h := fnv.New32a()
+	h.Write([]byte(hostname))
+	return "kube-apiserver-" + fmt.Sprint(h.Sum32())
+}
+
 func TestCreateLeaseOnStart(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
 	result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
@@ -50,6 +59,11 @@ func TestCreateLeaseOnStart(t *testing.T) {
 		t.Fatalf("Unexpected error: %v", err)
 	}
 
+	hostname, err := os.Hostname()
+	if err != nil {
+		t.Fatalf("Unexpected error getting apiserver hostname: %v", err)
+	}
+
 	t.Logf(`Waiting the kube-apiserver Lease to be created`)
 	if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
 		leases, err := kubeclient.
@@ -59,10 +73,25 @@ func TestCreateLeaseOnStart(t *testing.T) {
 		if err != nil {
 			return false, err
 		}
-		if leases != nil && len(leases.Items) == 1 && strings.HasPrefix(leases.Items[0].Name, "kube-apiserver-") {
+
-			return true, nil
+		if leases == nil {
+			return false, nil
 		}
-		return false, nil
+
+		if len(leases.Items) != 1 {
+			return false, nil
+		}
+
+		lease := leases.Items[0]
+		if lease.Name != expectedAPIServerIdentity(hostname) {
+			return false, fmt.Errorf("unexpected apiserver identity, got: %v, expected: %v", lease.Name, expectedAPIServerIdentity(hostname))
+		}
+
+		if lease.Labels[corev1.LabelHostname] != hostname {
+			return false, fmt.Errorf("unexpected hostname label, got: %v, expected: %v", lease.Labels[corev1.LabelHostname], hostname)
+		}
+
+		return true, nil
 	}); err != nil {
 		t.Fatalf("Failed to see the kube-apiserver lease: %v", err)
 	}