From a7de3e15a50bafdd65adf55f5fdc14567e3fa3e2 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Fri, 13 Jan 2023 15:49:30 -0500 Subject: [PATCH] apiserver: use the identity value in the apiserver identity hash Signed-off-by: Andrew Sy Kim --- pkg/controlplane/instance.go | 32 +++++++++++-------- .../src/k8s.io/apiserver/pkg/server/config.go | 19 ++++++++++- test/e2e/apimachinery/apiserver_identity.go | 14 +++++++- .../controlplane/apiserver_identity_test.go | 22 ++++++++++--- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index c5d872b35ca..b3c87c07bdc 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -507,7 +507,8 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) IdentityLeaseRenewIntervalPeriod, leaseName, metav1.NamespaceSystem, - labelAPIServerHeartbeat) + // TODO: receive identity label value as a parameter when post start hook is moved to generic apiserver. + labelAPIServerHeartbeatFunc(KubeAPIServer)) go controller.Run(hookContext.StopCh) return nil }) @@ -555,21 +556,24 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return m, nil } -func labelAPIServerHeartbeat(lease *coordinationapiv1.Lease) error { - if lease.Labels == nil { - lease.Labels = map[string]string{} - } - // This label indicates that kube-apiserver owns this identity lease object - lease.Labels[IdentityLeaseComponentLabelKey] = KubeAPIServer +func labelAPIServerHeartbeatFunc(identity string) lease.ProcessLeaseFunc { + return func(lease *coordinationapiv1.Lease) error { + if lease.Labels == nil { + lease.Labels = map[string]string{} + } - hostname, err := os.Hostname() - if err != nil { - return err - } + // This label indiciates the identity of the lease object. + lease.Labels[IdentityLeaseComponentLabelKey] = identity - // convenience label to easily map a lease object to a specific apiserver - lease.Labels[apiv1.LabelHostname] = hostname - return nil + hostname, err := os.Hostname() + if err != nil { + return err + } + + // convenience label to easily map a lease object to a specific apiserver + lease.Labels[apiv1.LabelHostname] = hostname + return nil + } } // InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled. diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index b9d5dc97722..86f8eca6d8e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -34,6 +34,7 @@ import ( jsonpatch "github.com/evanphx/json-patch" "github.com/google/uuid" + "golang.org/x/crypto/cryptobyte" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -345,7 +346,23 @@ func NewConfig(codecs serializer.CodecFactory) *Config { klog.Fatalf("error getting hostname for apiserver identity: %v", err) } - hash := sha256.Sum256([]byte(hostname)) + // Since the hash needs to be unique across each kube-apiserver and aggregated apiservers, + // the hash used for the identity should include both the hostname and the identity value. + // TODO: receive the identity value as a parameter once the apiserver identity lease controller + // post start hook is moved to generic apiserver. + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(hostname)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte("kube-apiserver")) + }) + hashData, err := b.Bytes() + if err != nil { + klog.Fatalf("error building hash data for apiserver identity: %v", err) + } + + hash := sha256.Sum256(hashData) id = "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16])) } lifecycleSignals := newLifecycleSignals() diff --git a/test/e2e/apimachinery/apiserver_identity.go b/test/e2e/apimachinery/apiserver_identity.go index 620aaf5dd52..c641b4a8b86 100644 --- a/test/e2e/apimachinery/apiserver_identity.go +++ b/test/e2e/apimachinery/apiserver_identity.go @@ -27,6 +27,8 @@ import ( "time" "github.com/onsi/ginkgo/v2" + "golang.org/x/crypto/cryptobyte" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -124,7 +126,17 @@ var _ = SIGDescribe("kube-apiserver identity [Feature:APIServerIdentity]", func( hostname, err := getControlPlaneHostname(ctx, &node) framework.ExpectNoError(err) - hash := sha256.Sum256([]byte(hostname)) + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(hostname)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte("kube-apiserver")) + }) + + hashData, err := b.Bytes() + framework.ExpectNoError(err) + hash := sha256.Sum256(hashData) leaseName := "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16])) lease, err := client.CoordinationV1().Leases(metav1.NamespaceSystem).Get(context.TODO(), leaseName, metav1.GetOptions{}) diff --git a/test/integration/controlplane/apiserver_identity_test.go b/test/integration/controlplane/apiserver_identity_test.go index 9f846fd2977..981a979a655 100644 --- a/test/integration/controlplane/apiserver_identity_test.go +++ b/test/integration/controlplane/apiserver_identity_test.go @@ -26,6 +26,8 @@ import ( "testing" "time" + "golang.org/x/crypto/cryptobyte" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -45,8 +47,20 @@ const ( testLeaseName = "apiserver-lease-test" ) -func expectedAPIServerIdentity(hostname string) string { - hash := sha256.Sum256([]byte(hostname)) +func expectedAPIServerIdentity(t *testing.T, hostname string) string { + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(hostname)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte("kube-apiserver")) + }) + hashData, err := b.Bytes() + if err != nil { + t.Fatalf("error building hash data for apiserver identity: %v", err) + } + + hash := sha256.Sum256(hashData) return "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16])) } @@ -84,8 +98,8 @@ func TestCreateLeaseOnStart(t *testing.T) { } lease := leases.Items[0] - if lease.Name != expectedAPIServerIdentity(hostname) { - return false, fmt.Errorf("unexpected apiserver identity, got: %v, expected: %v", lease.Name, expectedAPIServerIdentity(hostname)) + if lease.Name != expectedAPIServerIdentity(t, hostname) { + return false, fmt.Errorf("unexpected apiserver identity, got: %v, expected: %v", lease.Name, expectedAPIServerIdentity(t, hostname)) } if lease.Labels[corev1.LabelHostname] != hostname {