From 1f1dbc35a907a5fc01c954c22fe6c7a6d0f605e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanislav=20L=C3=A1zni=C4=8Dka?= Date: Wed, 6 Nov 2024 11:50:46 +0100 Subject: [PATCH] kubelet: ctb: use generics to handle alpha/beta APIs for CTB projection --- .../clustertrustbundle_manager.go | 173 ++++++--- .../clustertrustbundle_manager_test.go | 345 ++++++++++++++---- 2 files changed, 399 insertions(+), 119 deletions(-) diff --git a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go index 3c470f2a21c..6ad3aabedcb 100644 --- a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go +++ b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go @@ -27,15 +27,16 @@ import ( "time" "github.com/go-logr/logr" + certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" lrucache "k8s.io/apimachinery/pkg/util/cache" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" - certinformersv1beta1 "k8s.io/client-go/informers/certificates/v1beta1" clientset "k8s.io/client-go/kubernetes" - certlistersv1beta1 "k8s.io/client-go/listers/certificates/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" ) @@ -44,6 +45,51 @@ const ( maxLabelSelectorLength = 100 * 1024 ) +// clusterTrustBundle is a type constraint for version-independent ClusterTrustBundle API +type clusterTrustBundle interface { + certificatesv1alpha1.ClusterTrustBundle | certificatesv1beta1.ClusterTrustBundle +} + +// clusterTrustBundlesLister is an API-verion independent ClusterTrustBundles lister +type clusterTrustBundlesLister[T clusterTrustBundle] interface { + Get(string) (*T, error) + List(labels.Selector) ([]*T, error) +} + +type clusterTrustBundleHandlers[T clusterTrustBundle] interface { + GetName(*T) string + GetSignerName(*T) string + GetTrustBundle(*T) string +} + +type alphaClusterTrustBundleHandlers struct{} + +type betaClusterTrustBundleHandlers struct{} + +func (b *alphaClusterTrustBundleHandlers) GetName(ctb *certificatesv1alpha1.ClusterTrustBundle) string { + return ctb.Name +} + +func (b *alphaClusterTrustBundleHandlers) GetSignerName(ctb *certificatesv1alpha1.ClusterTrustBundle) string { + return ctb.Spec.SignerName +} + +func (b *alphaClusterTrustBundleHandlers) GetTrustBundle(ctb *certificatesv1alpha1.ClusterTrustBundle) string { + return ctb.Spec.TrustBundle +} + +func (b betaClusterTrustBundleHandlers) GetName(ctb *certificatesv1beta1.ClusterTrustBundle) string { + return ctb.Name +} + +func (b *betaClusterTrustBundleHandlers) GetSignerName(ctb *certificatesv1beta1.ClusterTrustBundle) string { + return ctb.Spec.SignerName +} + +func (b *betaClusterTrustBundleHandlers) GetTrustBundle(ctb *certificatesv1beta1.ClusterTrustBundle) string { + return ctb.Spec.TrustBundle +} + // Manager abstracts over the ability to get trust anchors. type Manager interface { GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) @@ -52,23 +98,44 @@ type Manager interface { // InformerManager is the "real" manager. It uses informers to track // ClusterTrustBundle objects. -type InformerManager struct { +type InformerManager[T clusterTrustBundle] struct { ctbInformer cache.SharedIndexInformer - ctbLister certlistersv1beta1.ClusterTrustBundleLister + ctbLister clusterTrustBundlesLister[T] + + ctbHandlers clusterTrustBundleHandlers[T] normalizationCache *lrucache.LRUExpireCache cacheTTL time.Duration } -var _ Manager = (*InformerManager)(nil) +var _ Manager = (*InformerManager[certificatesv1beta1.ClusterTrustBundle])(nil) -// NewInformerManager returns an initialized InformerManager. -func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.ClusterTrustBundleInformer, cacheSize int, cacheTTL time.Duration) (*InformerManager, error) { +func NewAlphaInformerManager( + ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration, +) (Manager, error) { + bundlesInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + return newInformerManager( + ctx, &alphaClusterTrustBundleHandlers{}, bundlesInformer.Informer(), bundlesInformer.Lister(), cacheSize, cacheTTL, + ) +} + +func NewBetaInformerManager( + ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration, +) (Manager, error) { + bundlesInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() + return newInformerManager( + ctx, &betaClusterTrustBundleHandlers{}, bundlesInformer.Informer(), bundlesInformer.Lister(), cacheSize, cacheTTL, + ) +} + +// newInformerManager returns an initialized InformerManager. +func newInformerManager[T clusterTrustBundle](ctx context.Context, handlers clusterTrustBundleHandlers[T], informer cache.SharedIndexInformer, lister clusterTrustBundlesLister[T], cacheSize int, cacheTTL time.Duration) (Manager, error) { // We need to call Informer() before calling start on the shared informer // factory, or the informer won't be registered to be started. - m := &InformerManager{ - ctbInformer: bundles.Informer(), - ctbLister: bundles.Lister(), + m := &InformerManager[T]{ + ctbInformer: informer, + ctbLister: lister, + ctbHandlers: handlers, normalizationCache: lrucache.NewLRUExpireCache(cacheSize), cacheTTL: cacheTTL, } @@ -78,34 +145,34 @@ func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.Cluste // apply to them. _, err := m.ctbInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { - ctb, ok := obj.(*certificatesv1beta1.ClusterTrustBundle) + ctb, ok := obj.(*T) if !ok { return } - logger.Info("Dropping all cache entries for signer", "signerName", ctb.Spec.SignerName) + logger.Info("Dropping all cache entries for signer", "signerName", m.ctbHandlers.GetSignerName(ctb)) m.dropCacheFor(ctb) }, UpdateFunc: func(old, new any) { - ctb, ok := new.(*certificatesv1beta1.ClusterTrustBundle) + ctb, ok := new.(*T) if !ok { return } - logger.Info("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName) - m.dropCacheFor(new.(*certificatesv1beta1.ClusterTrustBundle)) + logger.Info("Dropping cache for ClusterTrustBundle", "signerName", m.ctbHandlers.GetSignerName(ctb)) + m.dropCacheFor(new.(*T)) }, DeleteFunc: func(obj any) { - ctb, ok := obj.(*certificatesv1beta1.ClusterTrustBundle) + ctb, ok := obj.(*T) if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { return } - ctb, ok = tombstone.Obj.(*certificatesv1beta1.ClusterTrustBundle) + ctb, ok = tombstone.Obj.(*T) if !ok { return } } - logger.Info("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName) + logger.Info("Dropping cache for ClusterTrustBundle", "signerName", m.ctbHandlers.GetSignerName(ctb)) m.dropCacheFor(ctb) }, }) @@ -116,21 +183,21 @@ func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.Cluste return m, nil } -func (m *InformerManager) dropCacheFor(ctb *certificatesv1beta1.ClusterTrustBundle) { - if ctb.Spec.SignerName != "" { +func (m *InformerManager[T]) dropCacheFor(ctb *T) { + if ctbSignerName := m.ctbHandlers.GetSignerName(ctb); ctbSignerName != "" { m.normalizationCache.RemoveAll(func(key any) bool { - return key.(cacheKeyType).signerName == ctb.Spec.SignerName + return key.(cacheKeyType).signerName == ctbSignerName }) } else { m.normalizationCache.RemoveAll(func(key any) bool { - return key.(cacheKeyType).ctbName == ctb.ObjectMeta.Name + return key.(cacheKeyType).ctbName == m.ctbHandlers.GetName(ctb) }) } } // GetTrustAnchorsByName returns normalized and deduplicated trust anchors from // a single named ClusterTrustBundle. -func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { +func (m *InformerManager[T]) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { if !m.ctbInformer.HasSynced() { return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced") } @@ -149,7 +216,7 @@ func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool) return nil, fmt.Errorf("while getting ClusterTrustBundle: %w", err) } - pemTrustAnchors, err := m.normalizeTrustAnchors([]*certificatesv1beta1.ClusterTrustBundle{ctb}) + pemTrustAnchors, err := m.normalizeTrustAnchors([]*T{ctb}) if err != nil { return nil, fmt.Errorf("while normalizing trust anchors: %w", err) } @@ -161,7 +228,7 @@ func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool) // GetTrustAnchorsBySigner returns normalized and deduplicated trust anchors // from a set of selected ClusterTrustBundles. -func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { +func (m *InformerManager[T]) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { if !m.ctbInformer.HasSynced() { return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced") } @@ -188,9 +255,9 @@ func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelect return nil, fmt.Errorf("while listing ClusterTrustBundles matching label selector %v: %w", labelSelector, err) } - ctbList := []*certificatesv1beta1.ClusterTrustBundle{} + ctbList := []*T{} for _, ctb := range rawCTBList { - if ctb.Spec.SignerName == signerName { + if m.ctbHandlers.GetSignerName(ctb) == signerName { ctbList = append(ctbList, ctb) } } @@ -212,11 +279,11 @@ func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelect return pemTrustAnchors, nil } -func (m *InformerManager) normalizeTrustAnchors(ctbList []*certificatesv1beta1.ClusterTrustBundle) ([]byte, error) { +func (m *InformerManager[T]) normalizeTrustAnchors(ctbList []*T) ([]byte, error) { // Deduplicate trust anchors from all ClusterTrustBundles. trustAnchorSet := sets.Set[string]{} for _, ctb := range ctbList { - rest := []byte(ctb.Spec.TrustBundle) + rest := []byte(m.ctbHandlers.GetTrustBundle(ctb)) var b *pem.Block for { b, rest = pem.Decode(rest) @@ -309,6 +376,8 @@ func (m *LazyInformerManager) isManagerSet() bool { return m.manager != nil } +type managerConstructor func(ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration) (Manager, error) + func (m *LazyInformerManager) ensureManagerSet() error { if m.isManagerSet() { return nil @@ -321,24 +390,42 @@ func (m *LazyInformerManager) ensureManagerSet() error { return nil } - ctbAPIAvailable, err := clusterTrustBundlesAvailable(m.client) - if err != nil { - return fmt.Errorf("failed to determine which informer manager to choose: %w", err) - } - - if !ctbAPIAvailable { - m.manager = &NoopManager{} - return nil + managerSchema := map[schema.GroupVersion]managerConstructor{ + certificatesv1alpha1.SchemeGroupVersion: NewAlphaInformerManager, + certificatesv1beta1.SchemeGroupVersion: NewBetaInformerManager, } kubeInformers := informers.NewSharedInformerFactoryWithOptions(m.client, 0) - clusterTrustBundleManager, err := NewInformerManager(m.contextWithLogger, kubeInformers.Certificates().V1beta1().ClusterTrustBundles(), m.cacheSize, 5*time.Minute) - if err != nil { - return fmt.Errorf("error starting informer-based ClusterTrustBundle manager: %w", err) + + var clusterTrustBundleManager Manager + var foundGV string + for _, gv := range []schema.GroupVersion{certificatesv1beta1.SchemeGroupVersion, certificatesv1alpha1.SchemeGroupVersion} { + ctbAPIAvailable, err := clusterTrustBundlesAvailable(m.client, gv) + if err != nil { + return fmt.Errorf("failed to determine which informer manager to choose: %w", err) + } + + if !ctbAPIAvailable { + continue + } + + clusterTrustBundleManager, err = managerSchema[gv](m.contextWithLogger, kubeInformers, m.cacheSize, 5*time.Minute) + if err != nil { + return fmt.Errorf("error starting informer-based ClusterTrustBundle manager: %w", err) + } + foundGV = gv.String() + break } + + if clusterTrustBundleManager == nil { + m.manager = &NoopManager{} + m.logger.Info("No version of the ClusterTrustBundle API was found, the ClusterTrustBundle informer won't be started") + return nil + } + m.manager = clusterTrustBundleManager kubeInformers.Start(m.contextWithLogger.Done()) - m.logger.Info("Started ClusterTrustBundle informer") + m.logger.Info("Started ClusterTrustBundle informer", "apiGroup", foundGV) // a cache fetch will likely follow right after, wait for the freshly started // informers to sync @@ -358,8 +445,8 @@ func (m *LazyInformerManager) ensureManagerSet() error { return nil } -func clusterTrustBundlesAvailable(client clientset.Interface) (bool, error) { - resList, err := client.Discovery().ServerResourcesForGroupVersion(certificatesv1beta1.SchemeGroupVersion.String()) +func clusterTrustBundlesAvailable(client clientset.Interface, gv schema.GroupVersion) (bool, error) { + resList, err := client.Discovery().ServerResourcesForGroupVersion(gv.String()) if k8serrors.IsNotFound(err) { return false, nil } diff --git a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go index 636a0fa40f2..a78d29b4025 100644 --- a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go +++ b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go @@ -28,13 +28,23 @@ import ( "math/big" "sort" "strings" + "sync" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/test/utils/ktesting" @@ -46,8 +56,7 @@ func TestBeforeSynced(t *testing.T) { informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) - ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() - ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute) + ctbManager, _ := NewBetaInformerManager(tCtx, informerFactory, 256, 5*time.Minute) _, err := ctbManager.GetTrustAnchorsByName("foo", false) if err == nil { @@ -55,38 +64,74 @@ func TestBeforeSynced(t *testing.T) { } } +type testClient[T clusterTrustBundle] interface { + Create(context.Context, *T, metav1.CreateOptions) (*T, error) + Delete(context.Context, string, metav1.DeleteOptions) error +} + +// testingFunctionBundle is a API-version agnostic bundle of functions for handling CTBs in tests. +type testingFunctionBundle[T clusterTrustBundle] struct { + ctbConstructor func(name, signerName string, labels map[string]string, bundle string) *T + ctbToObj func(*T) runtime.Object + ctbTrustBundle func(*T) string + + informerManagerConstructor func(ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration) (Manager, error) + informerGetter func(informers.SharedInformerFactory) cache.SharedIndexInformer + clientGetter func(kubernetes.Interface) testClient[T] +} + +var alphaFunctionsBundle = testingFunctionBundle[certificatesv1alpha1.ClusterTrustBundle]{ + ctbConstructor: mustMakeAlphaCTB, + ctbToObj: func(ctb *certificatesv1alpha1.ClusterTrustBundle) runtime.Object { return ctb }, + ctbTrustBundle: (&alphaClusterTrustBundleHandlers{}).GetTrustBundle, + + informerManagerConstructor: NewAlphaInformerManager, + informerGetter: func(informerFactory informers.SharedInformerFactory) cache.SharedIndexInformer { + return informerFactory.Certificates().V1alpha1().ClusterTrustBundles().Informer() + }, + clientGetter: func(c kubernetes.Interface) testClient[certificatesv1alpha1.ClusterTrustBundle] { + return c.CertificatesV1alpha1().ClusterTrustBundles() + }, +} + +var betaFunctionsBundle = testingFunctionBundle[certificatesv1beta1.ClusterTrustBundle]{ + ctbConstructor: mustMakeBetaCTB, + ctbToObj: func(ctb *certificatesv1beta1.ClusterTrustBundle) runtime.Object { return ctb }, + ctbTrustBundle: (&betaClusterTrustBundleHandlers{}).GetTrustBundle, + + informerManagerConstructor: NewBetaInformerManager, + informerGetter: func(informerFactory informers.SharedInformerFactory) cache.SharedIndexInformer { + return informerFactory.Certificates().V1beta1().ClusterTrustBundles().Informer() + }, + clientGetter: func(c kubernetes.Interface) testClient[certificatesv1beta1.ClusterTrustBundle] { + return c.CertificatesV1beta1().ClusterTrustBundles() + }, +} + func TestGetTrustAnchorsByName(t *testing.T) { + t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsByName(t, alphaFunctionsBundle) }) + t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsByName(t, betaFunctionsBundle) }) +} + +func testGetTrustAnchorsByName[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) tCtx := ktesting.Init(t) defer cancel() - ctb1 := &certificatesv1beta1.ClusterTrustBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ctb1", - }, - Spec: certificatesv1beta1.ClusterTrustBundleSpec{ - TrustBundle: mustMakeRoot(t, "root1"), - }, - } + ctb1Bundle := mustMakeRoot(t, "root1") + ctb1 := b.ctbConstructor("ctb1", "", nil, ctb1Bundle) + ctb2Bundle := mustMakeRoot(t, "root2") + ctb2 := b.ctbConstructor("ctb2", "", nil, ctb2Bundle) - ctb2 := &certificatesv1beta1.ClusterTrustBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ctb2", - }, - Spec: certificatesv1beta1.ClusterTrustBundleSpec{ - TrustBundle: mustMakeRoot(t, "root2"), - }, - } - - kc := fake.NewSimpleClientset(ctb1, ctb2) + kc := fake.NewSimpleClientset(b.ctbToObj(ctb1), b.ctbToObj(ctb2)) informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) - ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() - ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute) + ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute) informerFactory.Start(ctx.Done()) - if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + ctbInformer := b.informerGetter(informerFactory) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) { t.Fatalf("Timed out waiting for informer to sync") } @@ -95,7 +140,7 @@ func TestGetTrustAnchorsByName(t *testing.T) { t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err) } - if diff := diffBundles(gotBundle, []byte(ctb1.Spec.TrustBundle)); diff != "" { + if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb1))); diff != "" { t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff) } @@ -104,7 +149,7 @@ func TestGetTrustAnchorsByName(t *testing.T) { t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err) } - if diff := diffBundles(gotBundle, []byte(ctb2.Spec.TrustBundle)); diff != "" { + if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb2))); diff != "" { t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff) } @@ -120,37 +165,30 @@ func TestGetTrustAnchorsByName(t *testing.T) { } func TestGetTrustAnchorsByNameCaching(t *testing.T) { + t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsByNameCaching(t, alphaFunctionsBundle) }) + t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsByNameCaching(t, betaFunctionsBundle) }) +} + +func testGetTrustAnchorsByNameCaching[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) { tCtx := ktesting.Init(t) ctx, cancel := context.WithTimeout(tCtx, 20*time.Second) defer cancel() - ctb1 := &certificatesv1beta1.ClusterTrustBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: certificatesv1beta1.ClusterTrustBundleSpec{ - TrustBundle: mustMakeRoot(t, "root1"), - }, - } + ctb1Bundle := mustMakeRoot(t, "root1") + ctb1 := b.ctbConstructor("foo", "", nil, ctb1Bundle) - ctb2 := &certificatesv1beta1.ClusterTrustBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: certificatesv1beta1.ClusterTrustBundleSpec{ - TrustBundle: mustMakeRoot(t, "root2"), - }, - } + ctb2Bundle := mustMakeRoot(t, "root2") + ctb2 := b.ctbConstructor("foo", "", nil, ctb2Bundle) - kc := fake.NewSimpleClientset(ctb1) + kc := fake.NewSimpleClientset(b.ctbToObj(ctb1)) informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) - ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() - ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute) + ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute) informerFactory.Start(ctx.Done()) - if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + ctbInformer := b.informerGetter(informerFactory) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) { t.Fatalf("Timed out waiting for informer to sync") } @@ -160,7 +198,7 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb1.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb1) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) @@ -173,17 +211,19 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb1.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb1) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } }) - if err := kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil { + client := b.clientGetter(kc) + + if err := client.Delete(ctx, "foo", metav1.DeleteOptions{}); err != nil { t.Fatalf("Error while deleting the old CTB: %v", err) } - if _, err := kc.CertificatesV1beta1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { + if _, err := client.Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { t.Fatalf("Error while adding new CTB: %v", err) } @@ -198,8 +238,7 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb2.Spec.TrustBundle - + wantBundle := b.ctbTrustBundle(ctb2) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } @@ -207,25 +246,30 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) { } func TestGetTrustAnchorsBySignerName(t *testing.T) { + t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsBySignerName(t, alphaFunctionsBundle) }) + t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsBySignerName(t, betaFunctionsBundle) }) +} + +func testGetTrustAnchorsBySignerName[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) tCtx := ktesting.Init(t) defer cancel() - ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) - ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) - ctb2dup := mustMakeCTB("signer-a-label-2-dup", "foo.bar/a", map[string]string{"label": "a"}, ctb2.Spec.TrustBundle) - ctb3 := mustMakeCTB("signer-a-label-b-1", "foo.bar/a", map[string]string{"label": "b"}, mustMakeRoot(t, "2")) - ctb4 := mustMakeCTB("signer-b-label-a-1", "foo.bar/b", map[string]string{"label": "a"}, mustMakeRoot(t, "3")) + ctb1 := b.ctbConstructor("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) + ctb2 := b.ctbConstructor("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) + ctb2dup := b.ctbConstructor("signer-a-label-2-dup", "foo.bar/a", map[string]string{"label": "a"}, b.ctbTrustBundle(ctb2)) + ctb3 := b.ctbConstructor("signer-a-label-b-1", "foo.bar/a", map[string]string{"label": "b"}, mustMakeRoot(t, "2")) + ctb4 := b.ctbConstructor("signer-b-label-a-1", "foo.bar/b", map[string]string{"label": "a"}, mustMakeRoot(t, "3")) - kc := fake.NewSimpleClientset(ctb1, ctb2, ctb2dup, ctb3, ctb4) + kc := fake.NewSimpleClientset(b.ctbToObj(ctb1), b.ctbToObj(ctb2), b.ctbToObj(ctb2dup), b.ctbToObj(ctb3), b.ctbToObj(ctb4)) informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) - ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() - ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute) + ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute) informerFactory.Start(ctx.Done()) - if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + ctbInformer := b.informerGetter(informerFactory) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) { t.Fatalf("Timed out waiting for informer to sync") } @@ -251,7 +295,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb1.Spec.TrustBundle + ctb2.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb1) + b.ctbTrustBundle(ctb2) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) @@ -277,7 +321,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" { + if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb4))); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } }) @@ -288,7 +332,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - if diff := diffBundles(gotBundle, []byte(ctb3.Spec.TrustBundle)); diff != "" { + if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb3))); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } }) @@ -299,7 +343,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" { + if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb4))); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } }) @@ -324,22 +368,27 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) { } func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) { + t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsBySignerNameCaching(t, alphaFunctionsBundle) }) + t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsBySignerNameCaching(t, betaFunctionsBundle) }) +} + +func testGetTrustAnchorsBySignerNameCaching[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) { tCtx := ktesting.Init(t) ctx, cancel := context.WithTimeout(tCtx, 20*time.Second) defer cancel() - ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) - ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) + ctb1 := b.ctbConstructor("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) + ctb2 := b.ctbConstructor("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) - kc := fake.NewSimpleClientset(ctb1) + kc := fake.NewSimpleClientset(b.ctbToObj(ctb1)) informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) - ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles() - ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute) + ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute) informerFactory.Start(ctx.Done()) - if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + ctbInformer := b.informerGetter(informerFactory) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) { t.Fatalf("Timed out waiting for informer to sync") } @@ -349,7 +398,7 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb1.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb1) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) @@ -362,17 +411,18 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb1.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb1) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) } }) - if err := kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil { + client := b.clientGetter(kc) + if err := client.Delete(ctx, "signer-a-label-a-1", metav1.DeleteOptions{}); err != nil { t.Fatalf("Error while deleting the old CTB: %v", err) } - if _, err := kc.CertificatesV1beta1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { + if _, err := client.Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { t.Fatalf("Error while adding new CTB: %v", err) } @@ -387,7 +437,7 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) { t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) } - wantBundle := ctb2.Spec.TrustBundle + wantBundle := b.ctbTrustBundle(ctb2) if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) @@ -422,7 +472,7 @@ func mustMakeRoot(t *testing.T, cn string) string { })) } -func mustMakeCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1beta1.ClusterTrustBundle { +func mustMakeBetaCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1beta1.ClusterTrustBundle { return &certificatesv1beta1.ClusterTrustBundle{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -435,6 +485,19 @@ func mustMakeCTB(name, signerName string, labels map[string]string, bundle strin } } +func mustMakeAlphaCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1alpha1.ClusterTrustBundle { + return &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: signerName, + TrustBundle: bundle, + }, + } +} + func diffBundles(a, b []byte) string { var block *pem.Block @@ -478,3 +541,133 @@ func diffBundles(a, b []byte) string { return cmp.Diff(aBlocks, bBlocks) } + +func TestLazyInformerManager_ensureManagerSet(t *testing.T) { + tests := []struct { + name string + injectError error + ctbsAvailableGVs []string + wantManager string + wantError bool + }{ + { + name: "API unavailable", + injectError: errors.NewNotFound(schema.GroupResource{Group: "certificates.k8s.io/v1beta1"}, ""), + wantManager: "noop", + }, + { + name: "err in discovery", + injectError: fmt.Errorf("unexpected discovery error"), + wantError: true, + wantManager: "nil", + }, + { + name: "API available in v1alpha1", + ctbsAvailableGVs: []string{"v1alpha1"}, + wantManager: "v1alpha1", + }, + { + name: "API available in an unhandled version", + ctbsAvailableGVs: []string{"v1beta2"}, + wantManager: "noop", + }, + { + name: "API available in v1beta1", + ctbsAvailableGVs: []string{"v1beta1"}, + wantManager: "v1beta1", + }, + { + name: "API available in v1 - currently unhandled", + ctbsAvailableGVs: []string{"v1"}, + wantManager: "noop", + }, + { + name: "err in discovery but beta API shard discovered", + injectError: fmt.Errorf("unexpected discovery error"), + ctbsAvailableGVs: []string{"v1beta1"}, + wantManager: "v1beta1", + }, + { + name: "API available in alpha and beta - prefer beta", + ctbsAvailableGVs: []string{"v1alpha1", "v1beta1"}, + wantManager: "v1beta1", + }, + { + name: "API available in multiple handled and unhandled versions - prefer the most-GA handled version", + ctbsAvailableGVs: []string{"v1alpha1", "v1", "v2", "v1beta1", "v1alpha2"}, + wantManager: "v1beta1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, loggerCtx := ktesting.NewTestContext(t) + + fakeDisc := fakeDiscovery{ + err: tt.injectError, + gvResources: make(map[string]*metav1.APIResourceList), + } + + for _, gv := range tt.ctbsAvailableGVs { + fakeDisc.gvResources["certificates.k8s.io/"+gv] = &metav1.APIResourceList{ + APIResources: []metav1.APIResource{ + {Name: "certificatesigningrequests"}, + {Name: "clustertrustbundles"}, + }, + } + } + + m := &LazyInformerManager{ + managerLock: sync.RWMutex{}, + client: NewFakeClientset(fakeDisc), + cacheSize: 128, + contextWithLogger: loggerCtx, + logger: logger, + } + if err := m.ensureManagerSet(); tt.wantError != (err != nil) { + t.Errorf("expected error: %t, got %v", tt.wantError, err) + } + + switch manager := m.manager.(type) { + case *InformerManager[certificatesv1alpha1.ClusterTrustBundle]: + require.Equal(t, tt.wantManager, "v1alpha1") + case *InformerManager[certificatesv1beta1.ClusterTrustBundle]: + require.Equal(t, tt.wantManager, "v1beta1") + case *NoopManager: + require.Equal(t, tt.wantManager, "noop") + case nil: + require.Equal(t, tt.wantManager, "nil") + default: + t.Fatalf("unknown manager type: %T", manager) + } + }) + } +} + +// fakeDiscovery inherits DiscoveryInterface(via FakeDiscovery) with some methods serving testing data. +type fakeDiscovery struct { + fakediscovery.FakeDiscovery + gvResources map[string]*metav1.APIResourceList + err error +} + +func (d fakeDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { + return d.gvResources[groupVersion], d.err +} + +type fakeDiscoveryClientSet struct { + *fake.Clientset + DiscoveryObj *fakeDiscovery +} + +func (c *fakeDiscoveryClientSet) Discovery() discovery.DiscoveryInterface { + return c.DiscoveryObj +} + +// Create a fake Clientset with its Discovery method overridden. +func NewFakeClientset(fakeDiscovery fakeDiscovery) *fakeDiscoveryClientSet { + cs := &fakeDiscoveryClientSet{ + Clientset: fake.NewClientset(), + DiscoveryObj: &fakeDiscovery, + } + return cs +}