kubelet: ctb: use generics to handle alpha/beta APIs for CTB projection

This commit is contained in:
Stanislav Láznička 2024-11-06 11:50:46 +01:00
parent 5b3b68a3a1
commit 1f1dbc35a9
No known key found for this signature in database
GPG Key ID: F8D8054395A1D157
2 changed files with 399 additions and 119 deletions

View File

@ -27,15 +27,16 @@ import (
"time"
"github.com/go-logr/logr"
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
lrucache "k8s.io/apimachinery/pkg/util/cache"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/informers"
certinformersv1beta1 "k8s.io/client-go/informers/certificates/v1beta1"
clientset "k8s.io/client-go/kubernetes"
certlistersv1beta1 "k8s.io/client-go/listers/certificates/v1beta1"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
)
@ -44,6 +45,51 @@ const (
maxLabelSelectorLength = 100 * 1024
)
// clusterTrustBundle is a type constraint for version-independent ClusterTrustBundle API
type clusterTrustBundle interface {
certificatesv1alpha1.ClusterTrustBundle | certificatesv1beta1.ClusterTrustBundle
}
// clusterTrustBundlesLister is an API-verion independent ClusterTrustBundles lister
type clusterTrustBundlesLister[T clusterTrustBundle] interface {
Get(string) (*T, error)
List(labels.Selector) ([]*T, error)
}
type clusterTrustBundleHandlers[T clusterTrustBundle] interface {
GetName(*T) string
GetSignerName(*T) string
GetTrustBundle(*T) string
}
type alphaClusterTrustBundleHandlers struct{}
type betaClusterTrustBundleHandlers struct{}
func (b *alphaClusterTrustBundleHandlers) GetName(ctb *certificatesv1alpha1.ClusterTrustBundle) string {
return ctb.Name
}
func (b *alphaClusterTrustBundleHandlers) GetSignerName(ctb *certificatesv1alpha1.ClusterTrustBundle) string {
return ctb.Spec.SignerName
}
func (b *alphaClusterTrustBundleHandlers) GetTrustBundle(ctb *certificatesv1alpha1.ClusterTrustBundle) string {
return ctb.Spec.TrustBundle
}
func (b betaClusterTrustBundleHandlers) GetName(ctb *certificatesv1beta1.ClusterTrustBundle) string {
return ctb.Name
}
func (b *betaClusterTrustBundleHandlers) GetSignerName(ctb *certificatesv1beta1.ClusterTrustBundle) string {
return ctb.Spec.SignerName
}
func (b *betaClusterTrustBundleHandlers) GetTrustBundle(ctb *certificatesv1beta1.ClusterTrustBundle) string {
return ctb.Spec.TrustBundle
}
// Manager abstracts over the ability to get trust anchors.
type Manager interface {
GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error)
@ -52,23 +98,44 @@ type Manager interface {
// InformerManager is the "real" manager. It uses informers to track
// ClusterTrustBundle objects.
type InformerManager struct {
type InformerManager[T clusterTrustBundle] struct {
ctbInformer cache.SharedIndexInformer
ctbLister certlistersv1beta1.ClusterTrustBundleLister
ctbLister clusterTrustBundlesLister[T]
ctbHandlers clusterTrustBundleHandlers[T]
normalizationCache *lrucache.LRUExpireCache
cacheTTL time.Duration
}
var _ Manager = (*InformerManager)(nil)
var _ Manager = (*InformerManager[certificatesv1beta1.ClusterTrustBundle])(nil)
// NewInformerManager returns an initialized InformerManager.
func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.ClusterTrustBundleInformer, cacheSize int, cacheTTL time.Duration) (*InformerManager, error) {
func NewAlphaInformerManager(
ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration,
) (Manager, error) {
bundlesInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles()
return newInformerManager(
ctx, &alphaClusterTrustBundleHandlers{}, bundlesInformer.Informer(), bundlesInformer.Lister(), cacheSize, cacheTTL,
)
}
func NewBetaInformerManager(
ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration,
) (Manager, error) {
bundlesInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
return newInformerManager(
ctx, &betaClusterTrustBundleHandlers{}, bundlesInformer.Informer(), bundlesInformer.Lister(), cacheSize, cacheTTL,
)
}
// newInformerManager returns an initialized InformerManager.
func newInformerManager[T clusterTrustBundle](ctx context.Context, handlers clusterTrustBundleHandlers[T], informer cache.SharedIndexInformer, lister clusterTrustBundlesLister[T], cacheSize int, cacheTTL time.Duration) (Manager, error) {
// We need to call Informer() before calling start on the shared informer
// factory, or the informer won't be registered to be started.
m := &InformerManager{
ctbInformer: bundles.Informer(),
ctbLister: bundles.Lister(),
m := &InformerManager[T]{
ctbInformer: informer,
ctbLister: lister,
ctbHandlers: handlers,
normalizationCache: lrucache.NewLRUExpireCache(cacheSize),
cacheTTL: cacheTTL,
}
@ -78,34 +145,34 @@ func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.Cluste
// apply to them.
_, err := m.ctbInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj any) {
ctb, ok := obj.(*certificatesv1beta1.ClusterTrustBundle)
ctb, ok := obj.(*T)
if !ok {
return
}
logger.Info("Dropping all cache entries for signer", "signerName", ctb.Spec.SignerName)
logger.Info("Dropping all cache entries for signer", "signerName", m.ctbHandlers.GetSignerName(ctb))
m.dropCacheFor(ctb)
},
UpdateFunc: func(old, new any) {
ctb, ok := new.(*certificatesv1beta1.ClusterTrustBundle)
ctb, ok := new.(*T)
if !ok {
return
}
logger.Info("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName)
m.dropCacheFor(new.(*certificatesv1beta1.ClusterTrustBundle))
logger.Info("Dropping cache for ClusterTrustBundle", "signerName", m.ctbHandlers.GetSignerName(ctb))
m.dropCacheFor(new.(*T))
},
DeleteFunc: func(obj any) {
ctb, ok := obj.(*certificatesv1beta1.ClusterTrustBundle)
ctb, ok := obj.(*T)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
return
}
ctb, ok = tombstone.Obj.(*certificatesv1beta1.ClusterTrustBundle)
ctb, ok = tombstone.Obj.(*T)
if !ok {
return
}
}
logger.Info("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName)
logger.Info("Dropping cache for ClusterTrustBundle", "signerName", m.ctbHandlers.GetSignerName(ctb))
m.dropCacheFor(ctb)
},
})
@ -116,21 +183,21 @@ func NewInformerManager(ctx context.Context, bundles certinformersv1beta1.Cluste
return m, nil
}
func (m *InformerManager) dropCacheFor(ctb *certificatesv1beta1.ClusterTrustBundle) {
if ctb.Spec.SignerName != "" {
func (m *InformerManager[T]) dropCacheFor(ctb *T) {
if ctbSignerName := m.ctbHandlers.GetSignerName(ctb); ctbSignerName != "" {
m.normalizationCache.RemoveAll(func(key any) bool {
return key.(cacheKeyType).signerName == ctb.Spec.SignerName
return key.(cacheKeyType).signerName == ctbSignerName
})
} else {
m.normalizationCache.RemoveAll(func(key any) bool {
return key.(cacheKeyType).ctbName == ctb.ObjectMeta.Name
return key.(cacheKeyType).ctbName == m.ctbHandlers.GetName(ctb)
})
}
}
// GetTrustAnchorsByName returns normalized and deduplicated trust anchors from
// a single named ClusterTrustBundle.
func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) {
func (m *InformerManager[T]) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) {
if !m.ctbInformer.HasSynced() {
return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced")
}
@ -149,7 +216,7 @@ func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool)
return nil, fmt.Errorf("while getting ClusterTrustBundle: %w", err)
}
pemTrustAnchors, err := m.normalizeTrustAnchors([]*certificatesv1beta1.ClusterTrustBundle{ctb})
pemTrustAnchors, err := m.normalizeTrustAnchors([]*T{ctb})
if err != nil {
return nil, fmt.Errorf("while normalizing trust anchors: %w", err)
}
@ -161,7 +228,7 @@ func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool)
// GetTrustAnchorsBySigner returns normalized and deduplicated trust anchors
// from a set of selected ClusterTrustBundles.
func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) {
func (m *InformerManager[T]) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) {
if !m.ctbInformer.HasSynced() {
return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced")
}
@ -188,9 +255,9 @@ func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelect
return nil, fmt.Errorf("while listing ClusterTrustBundles matching label selector %v: %w", labelSelector, err)
}
ctbList := []*certificatesv1beta1.ClusterTrustBundle{}
ctbList := []*T{}
for _, ctb := range rawCTBList {
if ctb.Spec.SignerName == signerName {
if m.ctbHandlers.GetSignerName(ctb) == signerName {
ctbList = append(ctbList, ctb)
}
}
@ -212,11 +279,11 @@ func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelect
return pemTrustAnchors, nil
}
func (m *InformerManager) normalizeTrustAnchors(ctbList []*certificatesv1beta1.ClusterTrustBundle) ([]byte, error) {
func (m *InformerManager[T]) normalizeTrustAnchors(ctbList []*T) ([]byte, error) {
// Deduplicate trust anchors from all ClusterTrustBundles.
trustAnchorSet := sets.Set[string]{}
for _, ctb := range ctbList {
rest := []byte(ctb.Spec.TrustBundle)
rest := []byte(m.ctbHandlers.GetTrustBundle(ctb))
var b *pem.Block
for {
b, rest = pem.Decode(rest)
@ -309,6 +376,8 @@ func (m *LazyInformerManager) isManagerSet() bool {
return m.manager != nil
}
type managerConstructor func(ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration) (Manager, error)
func (m *LazyInformerManager) ensureManagerSet() error {
if m.isManagerSet() {
return nil
@ -321,24 +390,42 @@ func (m *LazyInformerManager) ensureManagerSet() error {
return nil
}
ctbAPIAvailable, err := clusterTrustBundlesAvailable(m.client)
if err != nil {
return fmt.Errorf("failed to determine which informer manager to choose: %w", err)
}
if !ctbAPIAvailable {
m.manager = &NoopManager{}
return nil
managerSchema := map[schema.GroupVersion]managerConstructor{
certificatesv1alpha1.SchemeGroupVersion: NewAlphaInformerManager,
certificatesv1beta1.SchemeGroupVersion: NewBetaInformerManager,
}
kubeInformers := informers.NewSharedInformerFactoryWithOptions(m.client, 0)
clusterTrustBundleManager, err := NewInformerManager(m.contextWithLogger, kubeInformers.Certificates().V1beta1().ClusterTrustBundles(), m.cacheSize, 5*time.Minute)
if err != nil {
return fmt.Errorf("error starting informer-based ClusterTrustBundle manager: %w", err)
var clusterTrustBundleManager Manager
var foundGV string
for _, gv := range []schema.GroupVersion{certificatesv1beta1.SchemeGroupVersion, certificatesv1alpha1.SchemeGroupVersion} {
ctbAPIAvailable, err := clusterTrustBundlesAvailable(m.client, gv)
if err != nil {
return fmt.Errorf("failed to determine which informer manager to choose: %w", err)
}
if !ctbAPIAvailable {
continue
}
clusterTrustBundleManager, err = managerSchema[gv](m.contextWithLogger, kubeInformers, m.cacheSize, 5*time.Minute)
if err != nil {
return fmt.Errorf("error starting informer-based ClusterTrustBundle manager: %w", err)
}
foundGV = gv.String()
break
}
if clusterTrustBundleManager == nil {
m.manager = &NoopManager{}
m.logger.Info("No version of the ClusterTrustBundle API was found, the ClusterTrustBundle informer won't be started")
return nil
}
m.manager = clusterTrustBundleManager
kubeInformers.Start(m.contextWithLogger.Done())
m.logger.Info("Started ClusterTrustBundle informer")
m.logger.Info("Started ClusterTrustBundle informer", "apiGroup", foundGV)
// a cache fetch will likely follow right after, wait for the freshly started
// informers to sync
@ -358,8 +445,8 @@ func (m *LazyInformerManager) ensureManagerSet() error {
return nil
}
func clusterTrustBundlesAvailable(client clientset.Interface) (bool, error) {
resList, err := client.Discovery().ServerResourcesForGroupVersion(certificatesv1beta1.SchemeGroupVersion.String())
func clusterTrustBundlesAvailable(client clientset.Interface, gv schema.GroupVersion) (bool, error) {
resList, err := client.Discovery().ServerResourcesForGroupVersion(gv.String())
if k8serrors.IsNotFound(err) {
return false, nil
}

View File

@ -28,13 +28,23 @@ import (
"math/big"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
fakediscovery "k8s.io/client-go/discovery/fake"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/test/utils/ktesting"
@ -46,8 +56,7 @@ func TestBeforeSynced(t *testing.T) {
informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0)
ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute)
ctbManager, _ := NewBetaInformerManager(tCtx, informerFactory, 256, 5*time.Minute)
_, err := ctbManager.GetTrustAnchorsByName("foo", false)
if err == nil {
@ -55,38 +64,74 @@ func TestBeforeSynced(t *testing.T) {
}
}
type testClient[T clusterTrustBundle] interface {
Create(context.Context, *T, metav1.CreateOptions) (*T, error)
Delete(context.Context, string, metav1.DeleteOptions) error
}
// testingFunctionBundle is a API-version agnostic bundle of functions for handling CTBs in tests.
type testingFunctionBundle[T clusterTrustBundle] struct {
ctbConstructor func(name, signerName string, labels map[string]string, bundle string) *T
ctbToObj func(*T) runtime.Object
ctbTrustBundle func(*T) string
informerManagerConstructor func(ctx context.Context, informerFactory informers.SharedInformerFactory, cacheSize int, cacheTTL time.Duration) (Manager, error)
informerGetter func(informers.SharedInformerFactory) cache.SharedIndexInformer
clientGetter func(kubernetes.Interface) testClient[T]
}
var alphaFunctionsBundle = testingFunctionBundle[certificatesv1alpha1.ClusterTrustBundle]{
ctbConstructor: mustMakeAlphaCTB,
ctbToObj: func(ctb *certificatesv1alpha1.ClusterTrustBundle) runtime.Object { return ctb },
ctbTrustBundle: (&alphaClusterTrustBundleHandlers{}).GetTrustBundle,
informerManagerConstructor: NewAlphaInformerManager,
informerGetter: func(informerFactory informers.SharedInformerFactory) cache.SharedIndexInformer {
return informerFactory.Certificates().V1alpha1().ClusterTrustBundles().Informer()
},
clientGetter: func(c kubernetes.Interface) testClient[certificatesv1alpha1.ClusterTrustBundle] {
return c.CertificatesV1alpha1().ClusterTrustBundles()
},
}
var betaFunctionsBundle = testingFunctionBundle[certificatesv1beta1.ClusterTrustBundle]{
ctbConstructor: mustMakeBetaCTB,
ctbToObj: func(ctb *certificatesv1beta1.ClusterTrustBundle) runtime.Object { return ctb },
ctbTrustBundle: (&betaClusterTrustBundleHandlers{}).GetTrustBundle,
informerManagerConstructor: NewBetaInformerManager,
informerGetter: func(informerFactory informers.SharedInformerFactory) cache.SharedIndexInformer {
return informerFactory.Certificates().V1beta1().ClusterTrustBundles().Informer()
},
clientGetter: func(c kubernetes.Interface) testClient[certificatesv1beta1.ClusterTrustBundle] {
return c.CertificatesV1beta1().ClusterTrustBundles()
},
}
func TestGetTrustAnchorsByName(t *testing.T) {
t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsByName(t, alphaFunctionsBundle) })
t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsByName(t, betaFunctionsBundle) })
}
func testGetTrustAnchorsByName[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
tCtx := ktesting.Init(t)
defer cancel()
ctb1 := &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "ctb1",
},
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: mustMakeRoot(t, "root1"),
},
}
ctb1Bundle := mustMakeRoot(t, "root1")
ctb1 := b.ctbConstructor("ctb1", "", nil, ctb1Bundle)
ctb2Bundle := mustMakeRoot(t, "root2")
ctb2 := b.ctbConstructor("ctb2", "", nil, ctb2Bundle)
ctb2 := &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "ctb2",
},
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: mustMakeRoot(t, "root2"),
},
}
kc := fake.NewSimpleClientset(ctb1, ctb2)
kc := fake.NewSimpleClientset(b.ctbToObj(ctb1), b.ctbToObj(ctb2))
informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0)
ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute)
ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute)
informerFactory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) {
ctbInformer := b.informerGetter(informerFactory)
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) {
t.Fatalf("Timed out waiting for informer to sync")
}
@ -95,7 +140,7 @@ func TestGetTrustAnchorsByName(t *testing.T) {
t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err)
}
if diff := diffBundles(gotBundle, []byte(ctb1.Spec.TrustBundle)); diff != "" {
if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb1))); diff != "" {
t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff)
}
@ -104,7 +149,7 @@ func TestGetTrustAnchorsByName(t *testing.T) {
t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err)
}
if diff := diffBundles(gotBundle, []byte(ctb2.Spec.TrustBundle)); diff != "" {
if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb2))); diff != "" {
t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff)
}
@ -120,37 +165,30 @@ func TestGetTrustAnchorsByName(t *testing.T) {
}
func TestGetTrustAnchorsByNameCaching(t *testing.T) {
t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsByNameCaching(t, alphaFunctionsBundle) })
t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsByNameCaching(t, betaFunctionsBundle) })
}
func testGetTrustAnchorsByNameCaching[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) {
tCtx := ktesting.Init(t)
ctx, cancel := context.WithTimeout(tCtx, 20*time.Second)
defer cancel()
ctb1 := &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
},
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: mustMakeRoot(t, "root1"),
},
}
ctb1Bundle := mustMakeRoot(t, "root1")
ctb1 := b.ctbConstructor("foo", "", nil, ctb1Bundle)
ctb2 := &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
},
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: mustMakeRoot(t, "root2"),
},
}
ctb2Bundle := mustMakeRoot(t, "root2")
ctb2 := b.ctbConstructor("foo", "", nil, ctb2Bundle)
kc := fake.NewSimpleClientset(ctb1)
kc := fake.NewSimpleClientset(b.ctbToObj(ctb1))
informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0)
ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute)
ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute)
informerFactory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) {
ctbInformer := b.informerGetter(informerFactory)
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) {
t.Fatalf("Timed out waiting for informer to sync")
}
@ -160,7 +198,7 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb1.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb1)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
@ -173,17 +211,19 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb1.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb1)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
})
if err := kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil {
client := b.clientGetter(kc)
if err := client.Delete(ctx, "foo", metav1.DeleteOptions{}); err != nil {
t.Fatalf("Error while deleting the old CTB: %v", err)
}
if _, err := kc.CertificatesV1beta1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil {
if _, err := client.Create(ctx, ctb2, metav1.CreateOptions{}); err != nil {
t.Fatalf("Error while adding new CTB: %v", err)
}
@ -198,8 +238,7 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb2.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb2)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
@ -207,25 +246,30 @@ func TestGetTrustAnchorsByNameCaching(t *testing.T) {
}
func TestGetTrustAnchorsBySignerName(t *testing.T) {
t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsBySignerName(t, alphaFunctionsBundle) })
t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsBySignerName(t, betaFunctionsBundle) })
}
func testGetTrustAnchorsBySignerName[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
tCtx := ktesting.Init(t)
defer cancel()
ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0"))
ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1"))
ctb2dup := mustMakeCTB("signer-a-label-2-dup", "foo.bar/a", map[string]string{"label": "a"}, ctb2.Spec.TrustBundle)
ctb3 := mustMakeCTB("signer-a-label-b-1", "foo.bar/a", map[string]string{"label": "b"}, mustMakeRoot(t, "2"))
ctb4 := mustMakeCTB("signer-b-label-a-1", "foo.bar/b", map[string]string{"label": "a"}, mustMakeRoot(t, "3"))
ctb1 := b.ctbConstructor("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0"))
ctb2 := b.ctbConstructor("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1"))
ctb2dup := b.ctbConstructor("signer-a-label-2-dup", "foo.bar/a", map[string]string{"label": "a"}, b.ctbTrustBundle(ctb2))
ctb3 := b.ctbConstructor("signer-a-label-b-1", "foo.bar/a", map[string]string{"label": "b"}, mustMakeRoot(t, "2"))
ctb4 := b.ctbConstructor("signer-b-label-a-1", "foo.bar/b", map[string]string{"label": "a"}, mustMakeRoot(t, "3"))
kc := fake.NewSimpleClientset(ctb1, ctb2, ctb2dup, ctb3, ctb4)
kc := fake.NewSimpleClientset(b.ctbToObj(ctb1), b.ctbToObj(ctb2), b.ctbToObj(ctb2dup), b.ctbToObj(ctb3), b.ctbToObj(ctb4))
informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0)
ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute)
ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute)
informerFactory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) {
ctbInformer := b.informerGetter(informerFactory)
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) {
t.Fatalf("Timed out waiting for informer to sync")
}
@ -251,7 +295,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb1.Spec.TrustBundle + ctb2.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb1) + b.ctbTrustBundle(ctb2)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
@ -277,7 +321,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" {
if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb4))); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
})
@ -288,7 +332,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
if diff := diffBundles(gotBundle, []byte(ctb3.Spec.TrustBundle)); diff != "" {
if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb3))); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
})
@ -299,7 +343,7 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" {
if diff := diffBundles(gotBundle, []byte(b.ctbTrustBundle(ctb4))); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
})
@ -324,22 +368,27 @@ func TestGetTrustAnchorsBySignerName(t *testing.T) {
}
func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) {
t.Run("v1alpha1", func(t *testing.T) { testGetTrustAnchorsBySignerNameCaching(t, alphaFunctionsBundle) })
t.Run("v1beta1", func(t *testing.T) { testGetTrustAnchorsBySignerNameCaching(t, betaFunctionsBundle) })
}
func testGetTrustAnchorsBySignerNameCaching[T clusterTrustBundle](t *testing.T, b testingFunctionBundle[T]) {
tCtx := ktesting.Init(t)
ctx, cancel := context.WithTimeout(tCtx, 20*time.Second)
defer cancel()
ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0"))
ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1"))
ctb1 := b.ctbConstructor("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0"))
ctb2 := b.ctbConstructor("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1"))
kc := fake.NewSimpleClientset(ctb1)
kc := fake.NewSimpleClientset(b.ctbToObj(ctb1))
informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0)
ctbInformer := informerFactory.Certificates().V1beta1().ClusterTrustBundles()
ctbManager, _ := NewInformerManager(tCtx, ctbInformer, 256, 5*time.Minute)
ctbManager, _ := b.informerManagerConstructor(tCtx, informerFactory, 256, 5*time.Minute)
informerFactory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) {
ctbInformer := b.informerGetter(informerFactory)
if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.HasSynced) {
t.Fatalf("Timed out waiting for informer to sync")
}
@ -349,7 +398,7 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb1.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb1)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
@ -362,17 +411,18 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb1.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb1)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
}
})
if err := kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil {
client := b.clientGetter(kc)
if err := client.Delete(ctx, "signer-a-label-a-1", metav1.DeleteOptions{}); err != nil {
t.Fatalf("Error while deleting the old CTB: %v", err)
}
if _, err := kc.CertificatesV1beta1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil {
if _, err := client.Create(ctx, ctb2, metav1.CreateOptions{}); err != nil {
t.Fatalf("Error while adding new CTB: %v", err)
}
@ -387,7 +437,7 @@ func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) {
t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err)
}
wantBundle := ctb2.Spec.TrustBundle
wantBundle := b.ctbTrustBundle(ctb2)
if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" {
t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff)
@ -422,7 +472,7 @@ func mustMakeRoot(t *testing.T, cn string) string {
}))
}
func mustMakeCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1beta1.ClusterTrustBundle {
func mustMakeBetaCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1beta1.ClusterTrustBundle {
return &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: name,
@ -435,6 +485,19 @@ func mustMakeCTB(name, signerName string, labels map[string]string, bundle strin
}
}
func mustMakeAlphaCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1alpha1.ClusterTrustBundle {
return &certificatesv1alpha1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: labels,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
SignerName: signerName,
TrustBundle: bundle,
},
}
}
func diffBundles(a, b []byte) string {
var block *pem.Block
@ -478,3 +541,133 @@ func diffBundles(a, b []byte) string {
return cmp.Diff(aBlocks, bBlocks)
}
func TestLazyInformerManager_ensureManagerSet(t *testing.T) {
tests := []struct {
name string
injectError error
ctbsAvailableGVs []string
wantManager string
wantError bool
}{
{
name: "API unavailable",
injectError: errors.NewNotFound(schema.GroupResource{Group: "certificates.k8s.io/v1beta1"}, ""),
wantManager: "noop",
},
{
name: "err in discovery",
injectError: fmt.Errorf("unexpected discovery error"),
wantError: true,
wantManager: "nil",
},
{
name: "API available in v1alpha1",
ctbsAvailableGVs: []string{"v1alpha1"},
wantManager: "v1alpha1",
},
{
name: "API available in an unhandled version",
ctbsAvailableGVs: []string{"v1beta2"},
wantManager: "noop",
},
{
name: "API available in v1beta1",
ctbsAvailableGVs: []string{"v1beta1"},
wantManager: "v1beta1",
},
{
name: "API available in v1 - currently unhandled",
ctbsAvailableGVs: []string{"v1"},
wantManager: "noop",
},
{
name: "err in discovery but beta API shard discovered",
injectError: fmt.Errorf("unexpected discovery error"),
ctbsAvailableGVs: []string{"v1beta1"},
wantManager: "v1beta1",
},
{
name: "API available in alpha and beta - prefer beta",
ctbsAvailableGVs: []string{"v1alpha1", "v1beta1"},
wantManager: "v1beta1",
},
{
name: "API available in multiple handled and unhandled versions - prefer the most-GA handled version",
ctbsAvailableGVs: []string{"v1alpha1", "v1", "v2", "v1beta1", "v1alpha2"},
wantManager: "v1beta1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
logger, loggerCtx := ktesting.NewTestContext(t)
fakeDisc := fakeDiscovery{
err: tt.injectError,
gvResources: make(map[string]*metav1.APIResourceList),
}
for _, gv := range tt.ctbsAvailableGVs {
fakeDisc.gvResources["certificates.k8s.io/"+gv] = &metav1.APIResourceList{
APIResources: []metav1.APIResource{
{Name: "certificatesigningrequests"},
{Name: "clustertrustbundles"},
},
}
}
m := &LazyInformerManager{
managerLock: sync.RWMutex{},
client: NewFakeClientset(fakeDisc),
cacheSize: 128,
contextWithLogger: loggerCtx,
logger: logger,
}
if err := m.ensureManagerSet(); tt.wantError != (err != nil) {
t.Errorf("expected error: %t, got %v", tt.wantError, err)
}
switch manager := m.manager.(type) {
case *InformerManager[certificatesv1alpha1.ClusterTrustBundle]:
require.Equal(t, tt.wantManager, "v1alpha1")
case *InformerManager[certificatesv1beta1.ClusterTrustBundle]:
require.Equal(t, tt.wantManager, "v1beta1")
case *NoopManager:
require.Equal(t, tt.wantManager, "noop")
case nil:
require.Equal(t, tt.wantManager, "nil")
default:
t.Fatalf("unknown manager type: %T", manager)
}
})
}
}
// fakeDiscovery inherits DiscoveryInterface(via FakeDiscovery) with some methods serving testing data.
type fakeDiscovery struct {
fakediscovery.FakeDiscovery
gvResources map[string]*metav1.APIResourceList
err error
}
func (d fakeDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) {
return d.gvResources[groupVersion], d.err
}
type fakeDiscoveryClientSet struct {
*fake.Clientset
DiscoveryObj *fakeDiscovery
}
func (c *fakeDiscoveryClientSet) Discovery() discovery.DiscoveryInterface {
return c.DiscoveryObj
}
// Create a fake Clientset with its Discovery method overridden.
func NewFakeClientset(fakeDiscovery fakeDiscovery) *fakeDiscoveryClientSet {
cs := &fakeDiscoveryClientSet{
Clientset: fake.NewClientset(),
DiscoveryObj: &fakeDiscovery,
}
return cs
}