mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-13 05:02:50 +00:00
track legacy service account tokens
This commit is contained in:
Shihang Zhang
committed by
Shihang Zhang
Shihang Zhang
parent
7ad4b04632
commit
569cd70a52
211
pkg/controlplane/controller/legacytokentracking/controller.go
Normal file
211
pkg/controlplane/controller/legacytokentracking/controller.go
Normal file
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package legacytokentracking
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
corev1informers "k8s.io/client-go/informers/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/klog/v2"
|
||||
kubefeatures "k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/utils/clock"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfigMapName = "kube-apiserver-legacy-service-account-token-tracking"
|
||||
ConfigMapDataKey = "since"
|
||||
dateFormat = "2006-01-02"
|
||||
)
|
||||
|
||||
var (
|
||||
queueKey = metav1.NamespaceSystem + "/" + ConfigMapName
|
||||
)
|
||||
|
||||
// Controller maintains a timestamp value configmap `kube-apiserver-legacy-service-account-token-tracking`
|
||||
// in `kube-system` to indicates if the tracking for legacy tokens is enabled in
|
||||
// the cluster. For HA clusters, the configmap will be eventually created after
|
||||
// all controller instances have enabled the feature. When disabling this
|
||||
// feature, existing configmap will be deleted.
|
||||
type Controller struct {
|
||||
configMapClient corev1client.ConfigMapsGetter
|
||||
configMapInformer cache.SharedIndexInformer
|
||||
configMapCache cache.Indexer
|
||||
configMapSynced cache.InformerSynced
|
||||
queue workqueue.RateLimitingInterface
|
||||
|
||||
// enabled controls the behavior of the controller: if enabled is true, the
|
||||
//configmap will be created; otherwise, the configmap will be deleted.
|
||||
enabled bool
|
||||
// rate limiter controls the rate limit of the creation of the configmap.
|
||||
// this is useful in multi-apiserver cluster to prevent config existing in a
|
||||
// cluster with mixed enabled/disabled controllers. otherwise, those
|
||||
// apiservers will fight to create/delete until all apiservers are enabled
|
||||
// or disabled.
|
||||
creationRatelimiter *rate.Limiter
|
||||
clock clock.Clock
|
||||
}
|
||||
|
||||
// NewController returns a Controller struct.
|
||||
func NewController(cs kubernetes.Interface) *Controller {
|
||||
return newController(cs, clock.RealClock{}, rate.NewLimiter(rate.Every(30*time.Minute), 1))
|
||||
}
|
||||
|
||||
func newController(cs kubernetes.Interface, cl clock.Clock, limiter *rate.Limiter) *Controller {
|
||||
informer := corev1informers.NewFilteredConfigMapInformer(cs, metav1.NamespaceSystem, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(options *metav1.ListOptions) {
|
||||
options.FieldSelector = fields.OneTermEqualSelector("metadata.name", ConfigMapName).String()
|
||||
})
|
||||
|
||||
c := &Controller{
|
||||
configMapClient: cs.CoreV1(),
|
||||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "legacy_token_tracking_controller"),
|
||||
configMapInformer: informer,
|
||||
configMapCache: informer.GetIndexer(),
|
||||
configMapSynced: informer.HasSynced,
|
||||
enabled: utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenTracking),
|
||||
creationRatelimiter: limiter,
|
||||
clock: cl,
|
||||
}
|
||||
|
||||
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
c.enqueue()
|
||||
},
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
c.enqueue()
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
c.enqueue()
|
||||
},
|
||||
})
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Controller) enqueue() {
|
||||
c.queue.Add(queueKey)
|
||||
}
|
||||
|
||||
// Run starts the controller sync loop.
|
||||
func (c *Controller) Run(stopCh <-chan struct{}) {
|
||||
defer utilruntime.HandleCrash()
|
||||
defer c.queue.ShutDown()
|
||||
|
||||
klog.Info("Starting legacy_token_tracking_controller")
|
||||
defer klog.Infof("Shutting down legacy_token_tracking_controller")
|
||||
|
||||
go c.configMapInformer.Run(stopCh)
|
||||
if !cache.WaitForNamedCacheSync("configmaps", stopCh, c.configMapSynced) {
|
||||
return
|
||||
}
|
||||
|
||||
go wait.Until(c.runWorker, time.Second, stopCh)
|
||||
|
||||
c.queue.Add(queueKey)
|
||||
|
||||
<-stopCh
|
||||
klog.Info("Ending legacy_token_tracking_controller")
|
||||
}
|
||||
|
||||
func (c *Controller) runWorker() {
|
||||
for c.processNext() {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) processNext() bool {
|
||||
key, quit := c.queue.Get()
|
||||
if quit {
|
||||
return false
|
||||
}
|
||||
defer c.queue.Done(key)
|
||||
|
||||
if err := c.syncConfigMap(); err != nil {
|
||||
utilruntime.HandleError(fmt.Errorf("while syncing ConfigMap %q, err: %w", key, err))
|
||||
c.queue.AddRateLimited(key)
|
||||
return true
|
||||
}
|
||||
c.queue.Forget(key)
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) syncConfigMap() error {
|
||||
obj, exists, err := c.configMapCache.GetByKey(queueKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := c.clock.Now()
|
||||
switch {
|
||||
case c.enabled:
|
||||
if !exists {
|
||||
r := c.creationRatelimiter.ReserveN(now, 1)
|
||||
if delay := r.DelayFrom(now); delay > 0 {
|
||||
c.queue.AddAfter(queueKey, delay)
|
||||
r.CancelAt(now)
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err = c.configMapClient.ConfigMaps(metav1.NamespaceSystem).Create(context.TODO(), &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: now.UTC().Format(dateFormat)},
|
||||
}, metav1.CreateOptions{}); err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
// don't consume the creationRatelimiter for an unsuccessful attempt
|
||||
r.CancelAt(now)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
configMap := obj.(*corev1.ConfigMap)
|
||||
if _, err = time.Parse(dateFormat, configMap.Data[ConfigMapDataKey]); err != nil {
|
||||
configMap := configMap.DeepCopy()
|
||||
configMap.Data[ConfigMapDataKey] = now.UTC().Format(dateFormat)
|
||||
if _, err = c.configMapClient.ConfigMaps(metav1.NamespaceSystem).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil {
|
||||
if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case !c.enabled:
|
||||
if exists && obj.(*corev1.ConfigMap).DeletionTimestamp == nil {
|
||||
if err := c.configMapClient.ConfigMaps(metav1.NamespaceSystem).Delete(context.TODO(), ConfigMapName, metav1.DeleteOptions{}); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package legacytokentracking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"golang.org/x/time/rate"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
core "k8s.io/client-go/testing"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
testingclock "k8s.io/utils/clock/testing"
|
||||
)
|
||||
|
||||
const throttlePeriod = 30 * time.Second
|
||||
|
||||
func TestSyncConfigMap(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
tests := []struct {
|
||||
name string
|
||||
enabled bool
|
||||
nextCreateAt []time.Time
|
||||
clientObjects []runtime.Object
|
||||
existingConfigMap *corev1.ConfigMap
|
||||
|
||||
expectedErr error
|
||||
expectedActions []core.Action
|
||||
}{
|
||||
{
|
||||
name: "create configmap [no cache, no live object]",
|
||||
enabled: true,
|
||||
clientObjects: []runtime.Object{},
|
||||
expectedActions: []core.Action{
|
||||
core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "create configmap should ignore AlreadyExists error [no cache, live object exists]",
|
||||
enabled: true,
|
||||
clientObjects: []runtime.Object{
|
||||
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}},
|
||||
},
|
||||
expectedActions: []core.Action{
|
||||
core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "create configmap throttled [no cache, no live object]",
|
||||
enabled: true,
|
||||
nextCreateAt: []time.Time{now.Add(throttlePeriod - 2*time.Second), now.Add(throttlePeriod - time.Second)},
|
||||
clientObjects: []runtime.Object{},
|
||||
expectedActions: []core.Action{
|
||||
core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "create configmap after throttle period [no cache, no live object]",
|
||||
enabled: true,
|
||||
nextCreateAt: []time.Time{now.Add(throttlePeriod - 2*time.Second), now.Add(throttlePeriod - time.Second), now.Add(throttlePeriod + time.Second)},
|
||||
clientObjects: []runtime.Object{},
|
||||
expectedActions: []core.Action{
|
||||
core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Add(throttlePeriod + time.Second).Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "skip update configmap [cache with expected date format exists, live object exists]",
|
||||
enabled: true,
|
||||
clientObjects: []runtime.Object{
|
||||
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}},
|
||||
},
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)},
|
||||
},
|
||||
expectedActions: []core.Action{},
|
||||
},
|
||||
{
|
||||
name: "update configmap [cache with unexpected date format, live object exists]",
|
||||
enabled: true,
|
||||
clientObjects: []runtime.Object{
|
||||
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(time.RFC3339)}},
|
||||
},
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: now.Format(time.RFC3339)},
|
||||
},
|
||||
expectedActions: []core.Action{
|
||||
core.NewUpdateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "update configmap should ignore NotFound error [cache with unexpected date format, no live object]",
|
||||
enabled: true,
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: "BAD_TIMESTAMP"},
|
||||
},
|
||||
expectedActions: []core.Action{
|
||||
core.NewUpdateAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)}}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "delete configmap [no cache, no live object]",
|
||||
expectedActions: []core.Action{},
|
||||
},
|
||||
{
|
||||
name: "delete configmap [cache exists, live object exists]",
|
||||
clientObjects: []runtime.Object{
|
||||
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(time.RFC3339)}},
|
||||
},
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)},
|
||||
},
|
||||
expectedActions: []core.Action{
|
||||
core.NewDeleteAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, ConfigMapName),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "delete configmap that's alrady being deleted [cache exists, live object exists]",
|
||||
clientObjects: []runtime.Object{
|
||||
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName}, Data: map[string]string{ConfigMapDataKey: now.Format(time.RFC3339)}},
|
||||
},
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName, DeletionTimestamp: &metav1.Time{Time: time.Now()}},
|
||||
Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)},
|
||||
},
|
||||
expectedActions: []core.Action{},
|
||||
},
|
||||
{
|
||||
name: "delete configmap should ignore NotFound error [cache exists, no live object]",
|
||||
existingConfigMap: &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
Data: map[string]string{ConfigMapDataKey: now.Format(dateFormat)},
|
||||
},
|
||||
expectedActions: []core.Action{
|
||||
core.NewDeleteAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, ConfigMapName),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacyServiceAccountTokenTracking, test.enabled)()
|
||||
|
||||
client := fake.NewSimpleClientset(test.clientObjects...)
|
||||
limiter := rate.NewLimiter(rate.Every(throttlePeriod), 1)
|
||||
controller := newController(client, testingclock.NewFakeClock(now), limiter)
|
||||
if test.existingConfigMap != nil {
|
||||
controller.configMapCache.Add(test.existingConfigMap)
|
||||
}
|
||||
|
||||
if err := controller.syncConfigMap(); err != nil {
|
||||
t.Errorf("Failed to sync ConfigMap, err: %v", err)
|
||||
}
|
||||
|
||||
for _, createAt := range test.nextCreateAt {
|
||||
// delete the existing configmap to trigger second create
|
||||
controller.configMapCache.Delete(&corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: ConfigMapName},
|
||||
})
|
||||
controller.clock.(*testingclock.FakeClock).SetTime(createAt)
|
||||
if err := controller.syncConfigMap(); err != nil {
|
||||
t.Errorf("Failed to sync ConfigMap, err: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(test.expectedActions, client.Actions()); diff != "" {
|
||||
t.Errorf("Unexpected diff (-want +got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,7 @@ import (
|
||||
flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
|
||||
"k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc"
|
||||
"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
|
||||
"k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
|
||||
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
|
||||
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
|
||||
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
|
||||
@@ -496,6 +497,15 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
|
||||
})
|
||||
}
|
||||
|
||||
m.GenericAPIServer.AddPostStartHookOrDie("start-legacy-token-tracking-controller", func(hookContext genericapiserver.PostStartHookContext) error {
|
||||
kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go legacytokentracking.NewController(kubeClient).Run(hookContext.StopCh)
|
||||
return nil
|
||||
})
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user