From 21507902ba123c5c60eaa73436b95c4ae9b75908 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 24 Oct 2022 11:24:26 -0400 Subject: [PATCH 1/5] apiserver identity: use persistent identity format based on hostname Signed-off-by: Andrew Sy Kim --- pkg/controlplane/instance.go | 9 +++++++++ staging/src/k8s.io/apiserver/pkg/server/config.go | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 3dd15542957..a41dff33e6c 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "net/http" + "os" "reflect" "strconv" "time" @@ -515,6 +516,14 @@ func labelAPIServerHeartbeat(lease *coordinationapiv1.Lease) error { } // This label indicates that kube-apiserver owns this identity lease object lease.Labels[IdentityLeaseComponentLabelKey] = KubeAPIServer + + hostname, err := os.Hostname() + if err != nil { + return err + } + + // convenience label to easily map a lease object to a specific apiserver + lease.Labels[apiv1.LabelHostname] = hostname return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 27aeeeef292..26757f0437e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -19,8 +19,10 @@ package server import ( "context" "fmt" + "hash/fnv" "net" "net/http" + "os" goruntime "runtime" "runtime/debug" "sort" @@ -328,7 +330,14 @@ func NewConfig(codecs serializer.CodecFactory) *Config { defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz} var id string if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { - id = "kube-apiserver-" + uuid.New().String() + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("error getting hostname for apiserver identity: %v", err) + } + + h := fnv.New32a() + h.Write([]byte(hostname)) + id = "kube-apiserver-" + fmt.Sprint(h.Sum32()) } lifecycleSignals := newLifecycleSignals() From f74f819e1a295c9fbe1b02ea1c6b35ca05860c5f Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 24 Oct 2022 11:24:59 -0400 Subject: [PATCH 2/5] apiserver identity: update Lease creation integration test to validate new naming format and hostname label Signed-off-by: Andrew Sy Kim --- .../controlplane/apiserver_identity_test.go | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/integration/controlplane/apiserver_identity_test.go b/test/integration/controlplane/apiserver_identity_test.go index afe93b1bb29..4f918fa8195 100644 --- a/test/integration/controlplane/apiserver_identity_test.go +++ b/test/integration/controlplane/apiserver_identity_test.go @@ -18,11 +18,14 @@ package controlplane import ( "context" - "strings" + "fmt" + "hash/fnv" + "os" "testing" "time" coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -40,6 +43,12 @@ const ( testLeaseName = "apiserver-lease-test" ) +func expectedAPIServerIdentity(hostname string) string { + h := fnv.New32a() + h.Write([]byte(hostname)) + return "kube-apiserver-" + fmt.Sprint(h.Sum32()) +} + func TestCreateLeaseOnStart(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)() result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) @@ -50,6 +59,11 @@ func TestCreateLeaseOnStart(t *testing.T) { t.Fatalf("Unexpected error: %v", err) } + hostname, err := os.Hostname() + if err != nil { + t.Fatalf("Unexpected error getting apiserver hostname: %v", err) + } + t.Logf(`Waiting the kube-apiserver Lease to be created`) if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { leases, err := kubeclient. @@ -59,10 +73,25 @@ func TestCreateLeaseOnStart(t *testing.T) { if err != nil { return false, err } - if leases != nil && len(leases.Items) == 1 && strings.HasPrefix(leases.Items[0].Name, "kube-apiserver-") { - return true, nil + + if leases == nil { + return false, nil } - return false, nil + + if len(leases.Items) != 1 { + return false, nil + } + + lease := leases.Items[0] + if lease.Name != expectedAPIServerIdentity(hostname) { + return false, fmt.Errorf("unexpected apiserver identity, got: %v, expected: %v", lease.Name, expectedAPIServerIdentity(hostname)) + } + + if lease.Labels[corev1.LabelHostname] != hostname { + return false, fmt.Errorf("unexpected hostname label, got: %v, expected: %v", lease.Labels[corev1.LabelHostname], hostname) + } + + return true, nil }); err != nil { t.Fatalf("Failed to see the kube-apiserver lease: %v", err) } From 6925bee6d5c15c62cada8b7dab34e0a122cec8ce Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 24 Oct 2022 11:57:56 -0400 Subject: [PATCH 3/5] lease controller: add new constructor NewControllerWithLeaseName to support cases when the lease name and holder identity differ Signed-off-by: Andrew Sy Kim --- .../apimachinery/lease/controller.go | 25 ++++++++++++++++++- .../apimachinery/lease/controller_test.go | 2 ++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go index 3916bb1d7af..0602b786a7f 100644 --- a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go +++ b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go @@ -54,6 +54,7 @@ type controller struct { client clientset.Interface leaseClient coordclientset.LeaseInterface holderIdentity string + leaseName string leaseNamespace string leaseDurationSeconds int32 renewInterval time.Duration @@ -80,6 +81,28 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity client: client, leaseClient: leaseClient, holderIdentity: holderIdentity, + leaseName: holderIdentity, + leaseNamespace: leaseNamespace, + leaseDurationSeconds: leaseDurationSeconds, + renewInterval: renewInterval, + clock: clock, + onRepeatedHeartbeatFailure: onRepeatedHeartbeatFailure, + newLeasePostProcessFunc: newLeasePostProcessFunc, + } +} + +// NewControllerWithLeaseName is a copy of NewController but accepts a leaseName parameter. +// Use this constructor in cases when the lease name and holder identity should be different. +func NewControllerWithLeaseName(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseName, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller { + var leaseClient coordclientset.LeaseInterface + if client != nil { + leaseClient = client.CoordinationV1().Leases(leaseNamespace) + } + return &controller{ + client: client, + leaseClient: leaseClient, + holderIdentity: holderIdentity, + leaseName: leaseName, leaseNamespace: leaseNamespace, leaseDurationSeconds: leaseDurationSeconds, renewInterval: renewInterval, @@ -208,7 +231,7 @@ func (c *controller) newLease(base *coordinationv1.Lease) (*coordinationv1.Lease if base == nil { lease = &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: c.holderIdentity, + Name: c.leaseName, Namespace: c.leaseNamespace, }, Spec: coordinationv1.LeaseSpec{ diff --git a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller_test.go b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller_test.go index ce0c71edd2c..4d9cc01f49b 100644 --- a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller_test.go +++ b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller_test.go @@ -59,6 +59,7 @@ func TestNewNodeLease(t *testing.T) { desc: "nil base without node", controller: &controller{ client: fake.NewSimpleClientset(), + leaseName: node.Name, holderIdentity: node.Name, leaseDurationSeconds: 10, clock: fakeClock, @@ -80,6 +81,7 @@ func TestNewNodeLease(t *testing.T) { desc: "nil base with node", controller: &controller{ client: fake.NewSimpleClientset(node), + leaseName: node.Name, holderIdentity: node.Name, leaseDurationSeconds: 10, clock: fakeClock, From 3c0b75f4ad2668905d5ae30181f104060ac549a2 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 24 Oct 2022 11:58:34 -0400 Subject: [PATCH 4/5] apiserver identity: always use a unique value for the Lease holder identity Signed-off-by: Andrew Sy Kim --- pkg/controlplane/instance.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index a41dff33e6c..669ae582b4b 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -61,6 +61,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apiserver/pkg/endpoints/discovery" apiserverfeatures "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" @@ -471,13 +472,18 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) if err != nil { return err } - controller := lease.NewController( + + leaseName := m.GenericAPIServer.APIServerID + holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID()) + + controller := lease.NewControllerWithLeaseName( clock.RealClock{}, kubeClient, - m.GenericAPIServer.APIServerID, + holderIdentity, int32(c.ExtraConfig.IdentityLeaseDurationSeconds), nil, time.Duration(c.ExtraConfig.IdentityLeaseRenewIntervalSeconds)*time.Second, + leaseName, metav1.NamespaceSystem, labelAPIServerHeartbeat) go controller.Run(hookContext.StopCh) From 72f2e1cc0d543c942e318d3100386ffc6369f576 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Tue, 1 Nov 2022 12:25:01 -0400 Subject: [PATCH 5/5] lease controller: update NewController to accept leaseName as a parameter, remove NewControllerWithLeaseName Signed-off-by: Andrew Sy Kim --- pkg/controlplane/instance.go | 2 +- pkg/kubelet/kubelet.go | 1 + .../apimachinery/lease/controller.go | 25 ++----------------- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 669ae582b4b..d725263cfed 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -476,7 +476,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) leaseName := m.GenericAPIServer.APIServerID holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID()) - controller := lease.NewControllerWithLeaseName( + controller := lease.NewController( clock.RealClock{}, kubeClient, holderIdentity, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index a9660e711e6..6fdca955a20 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -833,6 +833,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeCfg.NodeLeaseDurationSeconds, klet.onRepeatedHeartbeatFailure, renewInterval, + string(klet.nodeName), v1.NamespaceNodeLease, util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName))) diff --git a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go index 0602b786a7f..203285e7442 100644 --- a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go +++ b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go @@ -72,28 +72,7 @@ type controller struct { } // NewController constructs and returns a controller -func NewController(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller { - var leaseClient coordclientset.LeaseInterface - if client != nil { - leaseClient = client.CoordinationV1().Leases(leaseNamespace) - } - return &controller{ - client: client, - leaseClient: leaseClient, - holderIdentity: holderIdentity, - leaseName: holderIdentity, - leaseNamespace: leaseNamespace, - leaseDurationSeconds: leaseDurationSeconds, - renewInterval: renewInterval, - clock: clock, - onRepeatedHeartbeatFailure: onRepeatedHeartbeatFailure, - newLeasePostProcessFunc: newLeasePostProcessFunc, - } -} - -// NewControllerWithLeaseName is a copy of NewController but accepts a leaseName parameter. -// Use this constructor in cases when the lease name and holder identity should be different. -func NewControllerWithLeaseName(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseName, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller { +func NewController(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseName, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller { var leaseClient coordclientset.LeaseInterface if client != nil { leaseClient = client.CoordinationV1().Leases(leaseNamespace) @@ -174,7 +153,7 @@ func (c *controller) backoffEnsureLease() (*coordinationv1.Lease, bool) { // ensureLease creates the lease if it does not exist. Returns the lease and // a bool (true if this call created the lease), or any error that occurs. func (c *controller) ensureLease() (*coordinationv1.Lease, bool, error) { - lease, err := c.leaseClient.Get(context.TODO(), c.holderIdentity, metav1.GetOptions{}) + lease, err := c.leaseClient.Get(context.TODO(), c.leaseName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { // lease does not exist, create it. leaseToCreate, err := c.newLease(nil)