diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index eba358b1805..eae41e55b04 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -1002,18 +1002,14 @@ func dropDisabledClusterTrustBundleProjection(podSpec, oldPodSpec *api.PodSpec) return } - for _, v := range podSpec.Volumes { - if v.Projected == nil { + for i := range podSpec.Volumes { + if podSpec.Volumes[i].Projected == nil { continue } - filteredSources := []api.VolumeProjection{} - for _, s := range v.Projected.Sources { - if s.ClusterTrustBundle == nil { - filteredSources = append(filteredSources, s) - } + for j := range podSpec.Volumes[i].Projected.Sources { + podSpec.Volumes[i].Projected.Sources[j].ClusterTrustBundle = nil } - v.Projected.Sources = filteredSources } } diff --git a/pkg/api/pod/util_test.go b/pkg/api/pod/util_test.go index ec33d2009b5..f27f7517d54 100644 --- a/pkg/api/pod/util_test.go +++ b/pkg/api/pod/util_test.go @@ -3237,3 +3237,156 @@ func TestMarkPodProposedForResize(t *testing.T) { }) } } + +func TestDropClusterTrustBundleProjectedVolumes(t *testing.T) { + testCases := []struct { + description string + clusterTrustBundleProjectionEnabled bool + oldPod *api.PodSpec + newPod *api.PodSpec + wantPod *api.PodSpec + }{ + { + description: "feature gate disabled, cannot add CTB volume to pod", + oldPod: &api.PodSpec{ + Volumes: []api.Volume{}, + }, + newPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + wantPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + {}, + }, + }}, + }, + }, + }, + }, + { + description: "feature gate disabled, can keep CTB volume on pod", + oldPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + newPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + wantPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + }, + { + description: "feature gate enabled, can add CTB volume to pod", + clusterTrustBundleProjectionEnabled: true, + oldPod: &api.PodSpec{ + Volumes: []api.Volume{}, + }, + newPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + wantPod: &api.PodSpec{ + Volumes: []api.Volume{ + { + Name: "foo", + VolumeSource: api.VolumeSource{ + Projected: &api.ProjectedVolumeSource{ + Sources: []api.VolumeProjection{ + { + ClusterTrustBundle: &api.ClusterTrustBundleProjection{ + Name: pointer.String("foo"), + }, + }, + }, + }}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClusterTrustBundleProjection, tc.clusterTrustBundleProjectionEnabled)() + + dropDisabledClusterTrustBundleProjection(tc.newPod, tc.oldPod) + if diff := cmp.Diff(tc.newPod, tc.wantPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + }) + } +} diff --git a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go new file mode 100644 index 00000000000..3f3337b7411 --- /dev/null +++ b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager.go @@ -0,0 +1,261 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package clustertrustbundle abstracts access to ClusterTrustBundles so that +// projected volumes can use them. +package clustertrustbundle + +import ( + "encoding/pem" + "fmt" + "math/rand" + "time" + + certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + lrucache "k8s.io/apimachinery/pkg/util/cache" + "k8s.io/apimachinery/pkg/util/sets" + certinformersv1alpha1 "k8s.io/client-go/informers/certificates/v1alpha1" + certlistersv1alpha1 "k8s.io/client-go/listers/certificates/v1alpha1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +const ( + maxLabelSelectorLength = 100 * 1024 +) + +// Manager abstracts over the ability to get trust anchors. +type Manager interface { + GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) + GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) +} + +// InformerManager is the "real" manager. It uses informers to track +// ClusterTrustBundle objects. +type InformerManager struct { + ctbInformer cache.SharedIndexInformer + ctbLister certlistersv1alpha1.ClusterTrustBundleLister + + normalizationCache *lrucache.LRUExpireCache + cacheTTL time.Duration +} + +var _ Manager = (*InformerManager)(nil) + +// NewInformerManager returns an initialized InformerManager. +func NewInformerManager(bundles certinformersv1alpha1.ClusterTrustBundleInformer, cacheSize int, cacheTTL time.Duration) (*InformerManager, error) { + // We need to call Informer() before calling start on the shared informer + // factory, or the informer won't be registered to be started. + m := &InformerManager{ + ctbInformer: bundles.Informer(), + ctbLister: bundles.Lister(), + normalizationCache: lrucache.NewLRUExpireCache(cacheSize), + cacheTTL: cacheTTL, + } + + // Have the informer bust cache entries when it sees updates that could + // apply to them. + _, err := m.ctbInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + ctb, ok := obj.(*certificatesv1alpha1.ClusterTrustBundle) + if !ok { + return + } + klog.InfoS("Dropping all cache entries for signer", "signerName", ctb.Spec.SignerName) + m.dropCacheFor(ctb) + }, + UpdateFunc: func(old, new any) { + ctb, ok := new.(*certificatesv1alpha1.ClusterTrustBundle) + if !ok { + return + } + klog.InfoS("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName) + m.dropCacheFor(new.(*certificatesv1alpha1.ClusterTrustBundle)) + }, + DeleteFunc: func(obj any) { + ctb, ok := obj.(*certificatesv1alpha1.ClusterTrustBundle) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + return + } + ctb, ok = tombstone.Obj.(*certificatesv1alpha1.ClusterTrustBundle) + if !ok { + return + } + } + klog.InfoS("Dropping cache for ClusterTrustBundle", "signerName", ctb.Spec.SignerName) + m.dropCacheFor(ctb) + }, + }) + if err != nil { + return nil, fmt.Errorf("while registering event handler on informer: %w", err) + } + + return m, nil +} + +func (m *InformerManager) dropCacheFor(ctb *certificatesv1alpha1.ClusterTrustBundle) { + if ctb.Spec.SignerName != "" { + m.normalizationCache.RemoveAll(func(key any) bool { + return key.(cacheKeyType).signerName == ctb.Spec.SignerName + }) + } else { + m.normalizationCache.RemoveAll(func(key any) bool { + return key.(cacheKeyType).ctbName == ctb.ObjectMeta.Name + }) + } +} + +// GetTrustAnchorsByName returns normalized and deduplicated trust anchors from +// a single named ClusterTrustBundle. +func (m *InformerManager) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { + if !m.ctbInformer.HasSynced() { + return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced") + } + + cacheKey := cacheKeyType{ctbName: name} + + if cachedAnchors, ok := m.normalizationCache.Get(cacheKey); ok { + return cachedAnchors.([]byte), nil + } + + ctb, err := m.ctbLister.Get(name) + if k8serrors.IsNotFound(err) && allowMissing { + return []byte{}, nil + } + if err != nil { + return nil, fmt.Errorf("while getting ClusterTrustBundle: %w", err) + } + + pemTrustAnchors, err := m.normalizeTrustAnchors([]*certificatesv1alpha1.ClusterTrustBundle{ctb}) + if err != nil { + return nil, fmt.Errorf("while normalizing trust anchors: %w", err) + } + + m.normalizationCache.Add(cacheKey, pemTrustAnchors, m.cacheTTL) + + return pemTrustAnchors, nil +} + +// GetTrustAnchorsBySigner returns normalized and deduplicated trust anchors +// from a set of selected ClusterTrustBundles. +func (m *InformerManager) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { + if !m.ctbInformer.HasSynced() { + return nil, fmt.Errorf("ClusterTrustBundle informer has not yet synced") + } + + // Note that this function treats nil as "match nothing", and non-nil but + // empty as "match everything". + selector, err := metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return nil, fmt.Errorf("while parsing label selector: %w", err) + } + + cacheKey := cacheKeyType{signerName: signerName, labelSelector: selector.String()} + + if lsLen := len(cacheKey.labelSelector); lsLen > maxLabelSelectorLength { + return nil, fmt.Errorf("label selector length (%d) is larger than %d", lsLen, maxLabelSelectorLength) + } + + if cachedAnchors, ok := m.normalizationCache.Get(cacheKey); ok { + return cachedAnchors.([]byte), nil + } + + rawCTBList, err := m.ctbLister.List(selector) + if err != nil { + return nil, fmt.Errorf("while listing ClusterTrustBundles matching label selector %v: %w", labelSelector, err) + } + + ctbList := []*certificatesv1alpha1.ClusterTrustBundle{} + for _, ctb := range rawCTBList { + if ctb.Spec.SignerName == signerName { + ctbList = append(ctbList, ctb) + } + } + + if len(ctbList) == 0 { + if allowMissing { + return []byte{}, nil + } + return nil, fmt.Errorf("combination of signerName and labelSelector matched zero ClusterTrustBundles") + } + + pemTrustAnchors, err := m.normalizeTrustAnchors(ctbList) + if err != nil { + return nil, fmt.Errorf("while normalizing trust anchors: %w", err) + } + + m.normalizationCache.Add(cacheKey, pemTrustAnchors, m.cacheTTL) + + return pemTrustAnchors, nil +} + +func (m *InformerManager) normalizeTrustAnchors(ctbList []*certificatesv1alpha1.ClusterTrustBundle) ([]byte, error) { + // Deduplicate trust anchors from all ClusterTrustBundles. + trustAnchorSet := sets.Set[string]{} + for _, ctb := range ctbList { + rest := []byte(ctb.Spec.TrustBundle) + var b *pem.Block + for { + b, rest = pem.Decode(rest) + if b == nil { + break + } + trustAnchorSet = trustAnchorSet.Insert(string(b.Bytes)) + } + } + + // Give the list a stable ordering that changes each time Kubelet restarts. + trustAnchorList := sets.List(trustAnchorSet) + rand.Shuffle(len(trustAnchorList), func(i, j int) { + trustAnchorList[i], trustAnchorList[j] = trustAnchorList[j], trustAnchorList[i] + }) + + pemTrustAnchors := []byte{} + for _, ta := range trustAnchorList { + b := &pem.Block{ + Type: "CERTIFICATE", + Bytes: []byte(ta), + } + pemTrustAnchors = append(pemTrustAnchors, pem.EncodeToMemory(b)...) + } + + return pemTrustAnchors, nil +} + +type cacheKeyType struct { + ctbName string + signerName string + labelSelector string +} + +// NoopManager always returns an error, for use in static kubelet mode. +type NoopManager struct{} + +var _ Manager = (*NoopManager)(nil) + +// GetTrustAnchorsByName implements Manager. +func (m *NoopManager) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { + return nil, fmt.Errorf("ClusterTrustBundle projection is not supported in static kubelet mode") +} + +// GetTrustAnchorsBySigner implements Manager. +func (m *NoopManager) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { + return nil, fmt.Errorf("ClusterTrustBundle projection is not supported in static kubelet mode") +} diff --git a/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go new file mode 100644 index 00000000000..eb778bdf8b5 --- /dev/null +++ b/pkg/kubelet/clustertrustbundle/clustertrustbundle_manager_test.go @@ -0,0 +1,474 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustertrustbundle + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "sort" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" +) + +func TestBeforeSynced(t *testing.T) { + kc := fake.NewSimpleClientset() + + informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) + + ctbInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + ctbManager, _ := NewInformerManager(ctbInformer, 256, 5*time.Minute) + + _, err := ctbManager.GetTrustAnchorsByName("foo", false) + if err == nil { + t.Fatalf("Got nil error, wanted non-nil") + } +} + +func TestGetTrustAnchorsByName(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ctb1 := &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ctb1", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: mustMakeRoot(t, "root1"), + }, + } + + ctb2 := &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ctb2", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: mustMakeRoot(t, "root2"), + }, + } + + kc := fake.NewSimpleClientset(ctb1, ctb2) + + informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) + + ctbInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + ctbManager, _ := NewInformerManager(ctbInformer, 256, 5*time.Minute) + + informerFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + t.Fatalf("Timed out waiting for informer to sync") + } + + gotBundle, err := ctbManager.GetTrustAnchorsByName("ctb1", false) + if err != nil { + t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err) + } + + if diff := diffBundles(gotBundle, []byte(ctb1.Spec.TrustBundle)); diff != "" { + t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff) + } + + gotBundle, err = ctbManager.GetTrustAnchorsByName("ctb2", false) + if err != nil { + t.Fatalf("Error while calling GetTrustAnchorsByName: %v", err) + } + + if diff := diffBundles(gotBundle, []byte(ctb2.Spec.TrustBundle)); diff != "" { + t.Fatalf("Got bad bundle; diff (-got +want)\n%s", diff) + } + + _, err = ctbManager.GetTrustAnchorsByName("not-found", false) + if err == nil { // EQUALS nil + t.Fatalf("While looking up nonexisting ClusterTrustBundle, got nil error, wanted non-nil") + } + + _, err = ctbManager.GetTrustAnchorsByName("not-found", true) + if err != nil { + t.Fatalf("Unexpected error while calling GetTrustAnchorsByName for nonexistent CTB with allowMissing: %v", err) + } +} + +func TestGetTrustAnchorsByNameCaching(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + ctb1 := &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: mustMakeRoot(t, "root1"), + }, + } + + ctb2 := &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: mustMakeRoot(t, "root2"), + }, + } + + kc := fake.NewSimpleClientset(ctb1) + + informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) + + ctbInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + ctbManager, _ := NewInformerManager(ctbInformer, 256, 5*time.Minute) + + informerFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + t.Fatalf("Timed out waiting for informer to sync") + } + + t.Run("foo should yield the first certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsByName("foo", false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb1.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("foo should still yield the first certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsByName("foo", false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb1.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + if err := kc.CertificatesV1alpha1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("Error while deleting the old CTB: %v", err) + } + if _, err := kc.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { + t.Fatalf("Error while adding new CTB: %v", err) + } + + // We need to sleep long enough for the informer to notice the new + // ClusterTrustBundle, but much less than the 5 minutes of the cache TTL. + // This shows us that the informer is properly clearing the cache. + time.Sleep(5 * time.Second) + + t.Run("foo should yield the new certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsByName("foo", false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb2.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) +} + +func TestGetTrustAnchorsBySignerName(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) + ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) + ctb2dup := mustMakeCTB("signer-a-label-2-dup", "foo.bar/a", map[string]string{"label": "a"}, ctb2.Spec.TrustBundle) + ctb3 := mustMakeCTB("signer-a-label-b-1", "foo.bar/a", map[string]string{"label": "b"}, mustMakeRoot(t, "2")) + ctb4 := mustMakeCTB("signer-b-label-a-1", "foo.bar/b", map[string]string{"label": "a"}, mustMakeRoot(t, "3")) + + kc := fake.NewSimpleClientset(ctb1, ctb2, ctb2dup, ctb3, ctb4) + + informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) + + ctbInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + ctbManager, _ := NewInformerManager(ctbInformer, 256, 5*time.Minute) + + informerFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + t.Fatalf("Timed out waiting for informer to sync") + } + + t.Run("big labelselector should cause error", func(t *testing.T) { + longString := strings.Builder{} + for i := 0; i < 63; i++ { + longString.WriteString("v") + } + matchLabels := map[string]string{} + for i := 0; i < 100*1024/63+1; i++ { + matchLabels[fmt.Sprintf("key-%d", i)] = longString.String() + } + + _, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: matchLabels}, false) + if err == nil || !strings.Contains(err.Error(), "label selector length") { + t.Fatalf("Bad error, got %v, wanted it to contain \"label selector length\"", err) + } + }) + + t.Run("signer-a label-a should yield two sorted certificates", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "a"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb1.Spec.TrustBundle + ctb2.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-a with nil selector should yield zero certificates", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", nil, true) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := "" + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-b with empty selector should yield one certificates", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/b", &metav1.LabelSelector{}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-a label-b should yield one certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "b"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + if diff := diffBundles(gotBundle, []byte(ctb3.Spec.TrustBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-b label-a should yield one certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/b", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "a"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + if diff := diffBundles(gotBundle, []byte(ctb4.Spec.TrustBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-b label-b allowMissing=true should yield zero certificates", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/b", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "b"}}, true) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + if diff := diffBundles(gotBundle, []byte{}); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-b label-b allowMissing=false should yield zero certificates (error)", func(t *testing.T) { + _, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/b", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "b"}}, false) + if err == nil { // EQUALS nil + t.Fatalf("Got nil error while calling GetTrustAnchorsBySigner, wanted non-nil") + } + }) +} + +func TestGetTrustAnchorsBySignerNameCaching(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + ctb1 := mustMakeCTB("signer-a-label-a-1", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "0")) + ctb2 := mustMakeCTB("signer-a-label-a-2", "foo.bar/a", map[string]string{"label": "a"}, mustMakeRoot(t, "1")) + + kc := fake.NewSimpleClientset(ctb1) + + informerFactory := informers.NewSharedInformerFactoryWithOptions(kc, 0) + + ctbInformer := informerFactory.Certificates().V1alpha1().ClusterTrustBundles() + ctbManager, _ := NewInformerManager(ctbInformer, 256, 5*time.Minute) + + informerFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), ctbInformer.Informer().HasSynced) { + t.Fatalf("Timed out waiting for informer to sync") + } + + t.Run("signer-a label-a should yield one certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "a"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb1.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + t.Run("signer-a label-a should yield the same result when called again", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "a"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb1.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) + + if err := kc.CertificatesV1alpha1().ClusterTrustBundles().Delete(ctx, ctb1.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("Error while deleting the old CTB: %v", err) + } + if _, err := kc.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, ctb2, metav1.CreateOptions{}); err != nil { + t.Fatalf("Error while adding new CTB: %v", err) + } + + // We need to sleep long enough for the informer to notice the new + // ClusterTrustBundle, but much less than the 5 minutes of the cache TTL. + // This shows us that the informer is properly clearing the cache. + time.Sleep(5 * time.Second) + + t.Run("signer-a label-a should return the new certificate", func(t *testing.T) { + gotBundle, err := ctbManager.GetTrustAnchorsBySigner("foo.bar/a", &metav1.LabelSelector{MatchLabels: map[string]string{"label": "a"}}, false) + if err != nil { + t.Fatalf("Got error while calling GetTrustAnchorsBySigner: %v", err) + } + + wantBundle := ctb2.Spec.TrustBundle + + if diff := diffBundles(gotBundle, []byte(wantBundle)); diff != "" { + t.Fatalf("Bad bundle; diff (-got +want)\n%s", diff) + } + }) +} + +func mustMakeRoot(t *testing.T, cn string) string { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("Error while generating key: %v", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + CommonName: cn, + }, + IsCA: true, + BasicConstraintsValid: true, + } + + cert, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) + if err != nil { + t.Fatalf("Error while making certificate: %v", err) + } + + return string(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: nil, + Bytes: cert, + })) +} + +func mustMakeCTB(name, signerName string, labels map[string]string, bundle string) *certificatesv1alpha1.ClusterTrustBundle { + return &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: signerName, + TrustBundle: bundle, + }, + } +} + +func diffBundles(a, b []byte) string { + var block *pem.Block + + aBlocks := []*pem.Block{} + for { + block, a = pem.Decode(a) + if block == nil { + break + } + aBlocks = append(aBlocks, block) + } + sort.Slice(aBlocks, func(i, j int) bool { + if aBlocks[i].Type < aBlocks[j].Type { + return true + } else if aBlocks[i].Type == aBlocks[j].Type { + comp := bytes.Compare(aBlocks[i].Bytes, aBlocks[j].Bytes) + return comp <= 0 + } else { + return false + } + }) + + bBlocks := []*pem.Block{} + for { + block, b = pem.Decode(b) + if block == nil { + break + } + bBlocks = append(bBlocks, block) + } + sort.Slice(bBlocks, func(i, j int) bool { + if bBlocks[i].Type < bBlocks[j].Type { + return true + } else if bBlocks[i].Type == bBlocks[j].Type { + comp := bytes.Compare(bBlocks[i].Bytes, bBlocks[j].Bytes) + return comp <= 0 + } else { + return false + } + }) + + return cmp.Diff(aBlocks, bBlocks) +} diff --git a/pkg/kubelet/config/common.go b/pkg/kubelet/config/common.go index 173c7bdd39f..69d6712623d 100644 --- a/pkg/kubelet/config/common.go +++ b/pkg/kubelet/config/common.go @@ -19,6 +19,7 @@ package config import ( "crypto/md5" "encoding/hex" + "errors" "fmt" "strings" @@ -102,6 +103,9 @@ func applyDefaults(pod *api.Pod, source string, isFile bool, nodeName types.Node type defaultFunc func(pod *api.Pod) error +// A static pod tried to use a ClusterTrustBundle projected volume source. +var ErrStaticPodTriedToUseClusterTrustBundle = errors.New("static pods may not use ClusterTrustBundle projected volume sources") + // tryDecodeSinglePod takes data and tries to extract valid Pod config information from it. func tryDecodeSinglePod(data []byte, defaultFn defaultFunc) (parsed bool, pod *v1.Pod, err error) { // JSON is valid YAML, so this should work for everything. @@ -136,6 +140,19 @@ func tryDecodeSinglePod(data []byte, defaultFn defaultFunc) (parsed bool, pod *v klog.ErrorS(err, "Pod failed to convert to v1", "pod", klog.KObj(newPod)) return true, nil, err } + + for _, v := range v1Pod.Spec.Volumes { + if v.Projected == nil { + continue + } + + for _, s := range v.Projected.Sources { + if s.ClusterTrustBundle != nil { + return true, nil, ErrStaticPodTriedToUseClusterTrustBundle + } + } + } + return true, v1Pod, nil } diff --git a/pkg/kubelet/config/common_test.go b/pkg/kubelet/config/common_test.go index 008fd8fc577..f390b6f95cb 100644 --- a/pkg/kubelet/config/common_test.go +++ b/pkg/kubelet/config/common_test.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "errors" "reflect" "testing" @@ -31,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/securitycontext" + "k8s.io/utils/ptr" ) func noDefault(*core.Pod) error { return nil } @@ -107,6 +109,76 @@ func TestDecodeSinglePod(t *testing.T) { } } +func TestDecodeSinglePodRejectsClusterTrustBundleVolumes(t *testing.T) { + grace := int64(30) + enableServiceLinks := v1.DefaultEnableServiceLinks + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: "12345", + Namespace: "mynamespace", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + TerminationGracePeriodSeconds: &grace, + Containers: []v1.Container{{ + Name: "image", + Image: "test/image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: v1.TerminationMessageReadFile, + SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(), + VolumeMounts: []v1.VolumeMount{ + { + Name: "ctb-volume", + MountPath: "/var/run/ctb-volume", + }, + }, + }}, + Volumes: []v1.Volume{ + { + Name: "ctb-volume", + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Name: ptr.To("my-ctb"), + Path: "ctb-file", + }, + }, + }, + }, + }, + }, + }, + SecurityContext: &v1.PodSecurityContext{}, + SchedulerName: v1.DefaultSchedulerName, + EnableServiceLinks: &enableServiceLinks, + }, + Status: v1.PodStatus{ + PodIP: "1.2.3.4", + PodIPs: []v1.PodIP{ + { + IP: "1.2.3.4", + }, + }, + }, + } + json, err := runtime.Encode(clientscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), pod) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + _, _, err = tryDecodeSinglePod(json, noDefault) + if !errors.Is(err, ErrStaticPodTriedToUseClusterTrustBundle) { + t.Errorf("Got error %q, want %q", err, ErrStaticPodTriedToUseClusterTrustBundle) + } +} + func TestDecodePodList(t *testing.T) { grace := int64(30) enableServiceLinks := v1.DefaultEnableServiceLinks diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 3b494893005..62083b62db4 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -75,6 +75,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/cloudresource" + "k8s.io/kubernetes/pkg/kubelet/clustertrustbundle" "k8s.io/kubernetes/pkg/kubelet/cm" draplugin "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin" "k8s.io/kubernetes/pkg/kubelet/config" @@ -451,7 +452,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, var serviceLister corelisters.ServiceLister var serviceHasSynced cache.InformerSynced if kubeDeps.KubeClient != nil { - kubeInformers := informers.NewSharedInformerFactory(kubeDeps.KubeClient, 0) + kubeInformers := informers.NewSharedInformerFactoryWithOptions(kubeDeps.KubeClient, 0) serviceLister = kubeInformers.Core().V1().Services().Lister() serviceHasSynced = kubeInformers.Core().V1().Services().Informer().HasSynced kubeInformers.Start(wait.NeverStop) @@ -793,11 +794,26 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tokenManager := token.NewManager(kubeDeps.KubeClient) + var clusterTrustBundleManager clustertrustbundle.Manager + if kubeDeps.KubeClient != nil && utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundleProjection) { + kubeInformers := informers.NewSharedInformerFactoryWithOptions(kubeDeps.KubeClient, 0) + clusterTrustBundleManager, err = clustertrustbundle.NewInformerManager(kubeInformers.Certificates().V1alpha1().ClusterTrustBundles(), 2*int(kubeCfg.MaxPods), 5*time.Minute) + if err != nil { + return nil, fmt.Errorf("while starting informer-based ClusterTrustBundle manager: %w", err) + } + kubeInformers.Start(wait.NeverStop) + klog.InfoS("Started ClusterTrustBundle informer") + } else { + // In static kubelet mode, use a no-op manager. + clusterTrustBundleManager = &clustertrustbundle.NoopManager{} + klog.InfoS("Not starting ClusterTrustBundle informer because we are in static kubelet mode") + } + // NewInitializedVolumePluginMgr initializes some storageErrors on the Kubelet runtimeState (in csi_plugin.go init) // which affects node ready status. This function must be called before Kubelet is initialized so that the Node // ReadyState is accurate with the storage state. klet.volumePluginMgr, err = - NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber) + NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, tokenManager, clusterTrustBundleManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber) if err != nil { return nil, err } diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 3da53e5ea85..09de07fa501 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -59,6 +59,7 @@ import ( "k8s.io/kubernetes/pkg/features" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" + "k8s.io/kubernetes/pkg/kubelet/clustertrustbundle" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/configmap" @@ -379,7 +380,7 @@ func newTestKubeletWithImageList( var prober volume.DynamicPluginProber // TODO (#51147) inject mock kubelet.volumePluginMgr, err = - NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, token.NewManager(kubelet.kubeClient), allPlugins, prober) + NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, token.NewManager(kubelet.kubeClient), &clustertrustbundle.NoopManager{}, allPlugins, prober) require.NoError(t, err, "Failed to initialize VolumePluginMgr") kubelet.volumeManager = kubeletvolume.NewVolumeManager( diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index f8d0d972585..5d69df087ec 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/client-go/tools/record" utiltesting "k8s.io/client-go/util/testing" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" + "k8s.io/kubernetes/pkg/kubelet/clustertrustbundle" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/configmap" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -72,6 +73,7 @@ func TestRunOnce(t *testing.T) { }, nil).AnyTimes() fakeSecretManager := secret.NewFakeManager() fakeConfigMapManager := configmap.NewFakeManager() + clusterTrustBundleManager := &clustertrustbundle.NoopManager{} podManager := kubepod.NewBasicPodManager() fakeRuntime := &containertest.FakeRuntime{} podStartupLatencyTracker := kubeletutil.NewPodStartupLatencyTracker() @@ -103,7 +105,7 @@ func TestRunOnce(t *testing.T) { plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil} kb.volumePluginMgr, err = - NewInitializedVolumePluginMgr(kb, fakeSecretManager, fakeConfigMapManager, nil, []volume.VolumePlugin{plug}, nil /* prober */) + NewInitializedVolumePluginMgr(kb, fakeSecretManager, fakeConfigMapManager, nil, clusterTrustBundleManager, []volume.VolumePlugin{plug}, nil /* prober */) if err != nil { t.Fatalf("failed to initialize VolumePluginMgr: %v", err) } diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 7a9a9f87122..b2fe38634ef 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -27,6 +27,7 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" @@ -35,6 +36,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" + "k8s.io/kubernetes/pkg/kubelet/clustertrustbundle" "k8s.io/kubernetes/pkg/kubelet/configmap" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/token" @@ -55,6 +57,7 @@ func NewInitializedVolumePluginMgr( secretManager secret.Manager, configMapManager configmap.Manager, tokenManager *token.Manager, + clusterTrustBundleManager clustertrustbundle.Manager, plugins []volume.VolumePlugin, prober volume.DynamicPluginProber) (*volume.VolumePluginMgr, error) { @@ -75,15 +78,16 @@ func NewInitializedVolumePluginMgr( } kvh := &kubeletVolumeHost{ - kubelet: kubelet, - volumePluginMgr: volume.VolumePluginMgr{}, - secretManager: secretManager, - configMapManager: configMapManager, - tokenManager: tokenManager, - informerFactory: informerFactory, - csiDriverLister: csiDriverLister, - csiDriversSynced: csiDriversSynced, - exec: utilexec.New(), + kubelet: kubelet, + volumePluginMgr: volume.VolumePluginMgr{}, + secretManager: secretManager, + configMapManager: configMapManager, + tokenManager: tokenManager, + clusterTrustBundleManager: clusterTrustBundleManager, + informerFactory: informerFactory, + csiDriverLister: csiDriverLister, + csiDriversSynced: csiDriversSynced, + exec: utilexec.New(), } if err := kvh.volumePluginMgr.InitPlugins(plugins, prober, kvh); err != nil { @@ -104,15 +108,16 @@ func (kvh *kubeletVolumeHost) GetPluginDir(pluginName string) string { } type kubeletVolumeHost struct { - kubelet *Kubelet - volumePluginMgr volume.VolumePluginMgr - secretManager secret.Manager - tokenManager *token.Manager - configMapManager configmap.Manager - informerFactory informers.SharedInformerFactory - csiDriverLister storagelisters.CSIDriverLister - csiDriversSynced cache.InformerSynced - exec utilexec.Interface + kubelet *Kubelet + volumePluginMgr volume.VolumePluginMgr + secretManager secret.Manager + tokenManager *token.Manager + configMapManager configmap.Manager + clusterTrustBundleManager clustertrustbundle.Manager + informerFactory informers.SharedInformerFactory + csiDriverLister storagelisters.CSIDriverLister + csiDriversSynced cache.InformerSynced + exec utilexec.Interface } func (kvh *kubeletVolumeHost) SetKubeletError(err error) { @@ -266,6 +271,14 @@ func (kvh *kubeletVolumeHost) DeleteServiceAccountTokenFunc() func(podUID types. return kvh.tokenManager.DeleteServiceAccountToken } +func (kvh *kubeletVolumeHost) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { + return kvh.clusterTrustBundleManager.GetTrustAnchorsByName(name, allowMissing) +} + +func (kvh *kubeletVolumeHost) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { + return kvh.clusterTrustBundleManager.GetTrustAnchorsBySigner(signerName, labelSelector, allowMissing) +} + func (kvh *kubeletVolumeHost) GetNodeLabels() (map[string]string, error) { node, err := kvh.kubelet.GetNode() if err != nil { diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 14cee160a63..94c2330afc9 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -333,6 +333,13 @@ type KubeletVolumeHost interface { WaitForCacheSync() error // Returns hostutil.HostUtils GetHostUtil() hostutil.HostUtils + + // Returns trust anchors from the named ClusterTrustBundle. + GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) + + // Returns trust anchors from the ClusterTrustBundles selected by signer + // name and label selector. + GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) } // AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go index deb7728168a..01e5c1e3e38 100644 --- a/pkg/volume/projected/projected.go +++ b/pkg/volume/projected/projected.go @@ -45,6 +45,7 @@ const ( type projectedPlugin struct { host volume.VolumeHost + kvHost volume.KubeletVolumeHost getSecret func(namespace, name string) (*v1.Secret, error) getConfigMap func(namespace, name string) (*v1.ConfigMap, error) getServiceAccountToken func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) @@ -69,6 +70,7 @@ func getPath(uid types.UID, volName string, host volume.VolumeHost) string { func (plugin *projectedPlugin) Init(host volume.VolumeHost) error { plugin.host = host + plugin.kvHost = host.(volume.KubeletVolumeHost) plugin.getSecret = host.GetSecretFunc() plugin.getConfigMap = host.GetConfigMapFunc() plugin.getServiceAccountToken = host.GetServiceAccountTokenFunc() @@ -353,6 +355,42 @@ func (s *projectedVolumeMounter) collectData(mounterArgs volume.MounterArgs) (ma Mode: mode, FsUser: mounterArgs.FsUser, } + case source.ClusterTrustBundle != nil: + allowEmpty := false + if source.ClusterTrustBundle.Optional != nil && *source.ClusterTrustBundle.Optional { + allowEmpty = true + } + + var trustAnchors []byte + if source.ClusterTrustBundle.Name != nil { + var err error + trustAnchors, err = s.plugin.kvHost.GetTrustAnchorsByName(*source.ClusterTrustBundle.Name, allowEmpty) + if err != nil { + errlist = append(errlist, err) + continue + } + } else if source.ClusterTrustBundle.SignerName != nil { + var err error + trustAnchors, err = s.plugin.kvHost.GetTrustAnchorsBySigner(*source.ClusterTrustBundle.SignerName, source.ClusterTrustBundle.LabelSelector, allowEmpty) + if err != nil { + errlist = append(errlist, err) + continue + } + } else { + errlist = append(errlist, fmt.Errorf("ClusterTrustBundle projection requires either name or signerName to be set")) + continue + } + + mode := *s.source.DefaultMode + if mounterArgs.FsUser != nil || mounterArgs.FsGroup != nil { + mode = 0600 + } + + payload[source.ClusterTrustBundle.Path] = volumeutil.FileProjection{ + Data: trustAnchors, + Mode: mode, + FsUser: mounterArgs.FsUser, + } } } return payload, utilerrors.NewAggregate(errlist) diff --git a/pkg/volume/projected/projected_test.go b/pkg/volume/projected/projected_test.go index 880755b2649..e785ae15bf3 100644 --- a/pkg/volume/projected/projected_test.go +++ b/pkg/volume/projected/projected_test.go @@ -17,7 +17,13 @@ limitations under the License. package projected import ( + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" + "math/big" "os" "path/filepath" "reflect" @@ -26,6 +32,7 @@ import ( "github.com/google/go-cmp/cmp" authenticationv1 "k8s.io/api/authentication/v1" + certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -872,13 +879,172 @@ func TestCollectDataWithServiceAccountToken(t *testing.T) { } } +func TestCollectDataWithClusterTrustBundle(t *testing.T) { + // This test is limited by the use of a fake clientset and volume host. We + // can't meaningfully test that label selectors end up doing the correct + // thing for example. + + goodCert1 := mustMakeRoot(t, "root1") + + testCases := []struct { + name string + + source v1.ProjectedVolumeSource + bundles []runtime.Object + + fsUser *int64 + fsGroup *int64 + + wantPayload map[string]util.FileProjection + wantErr error + }{ + { + name: "single ClusterTrustBundle by name", + source: v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Name: utilptr.String("foo"), + Path: "bundle.pem", + }, + }, + }, + DefaultMode: utilptr.Int32(0644), + }, + bundles: []runtime.Object{ + &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: string(goodCert1), + }, + }, + }, + wantPayload: map[string]util.FileProjection{ + "bundle.pem": { + Data: []byte(goodCert1), + Mode: 0644, + }, + }, + }, + { + name: "single ClusterTrustBundle by signer name", + source: v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + SignerName: utilptr.String("foo.example/bar"), // Note: fake client doesn't understand selection by signer name. + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key": "non-value", // Note: fake client doesn't actually act on label selectors. + }, + }, + Path: "bundle.pem", + }, + }, + }, + DefaultMode: utilptr.Int32(0644), + }, + bundles: []runtime.Object{ + &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo:example:bar", + Labels: map[string]string{ + "key": "value", + }, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: "foo.example/bar", + TrustBundle: string(goodCert1), + }, + }, + }, + wantPayload: map[string]util.FileProjection{ + "bundle.pem": { + Data: []byte(goodCert1), + Mode: 0644, + }, + }, + }, + { + name: "single ClusterTrustBundle by name, non-default mode", + source: v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Name: utilptr.String("foo"), + Path: "bundle.pem", + }, + }, + }, + DefaultMode: utilptr.Int32(0600), + }, + bundles: []runtime.Object{ + &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: string(goodCert1), + }, + }, + }, + wantPayload: map[string]util.FileProjection{ + "bundle.pem": { + Data: []byte(goodCert1), + Mode: 0600, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + UID: types.UID("test_pod_uid"), + }, + Spec: v1.PodSpec{ServiceAccountName: "foo"}, + } + + client := fake.NewSimpleClientset(tc.bundles...) + + tempDir, host := newTestHost(t, client) + defer os.RemoveAll(tempDir) + + var myVolumeMounter = projectedVolumeMounter{ + projectedVolume: &projectedVolume{ + sources: tc.source.Sources, + podUID: pod.UID, + plugin: &projectedPlugin{ + host: host, + kvHost: host.(volume.KubeletVolumeHost), + }, + }, + source: tc.source, + pod: pod, + } + + gotPayload, err := myVolumeMounter.collectData(volume.MounterArgs{FsUser: tc.fsUser, FsGroup: tc.fsGroup}) + if err != nil { + t.Fatalf("Unexpected failure making payload: %v", err) + } + if diff := cmp.Diff(tc.wantPayload, gotPayload); diff != "" { + t.Fatalf("Bad payload; diff (-want +got)\n%s", diff) + } + }) + } +} + func newTestHost(t *testing.T, clientset clientset.Interface) (string, volume.VolumeHost) { tempDir, err := os.MkdirTemp("", "projected_volume_test.") if err != nil { t.Fatalf("can't make a temp rootdir: %v", err) } - return tempDir, volumetest.NewFakeVolumeHost(t, tempDir, clientset, emptydir.ProbeVolumePlugins()) + return tempDir, volumetest.NewFakeKubeletVolumeHost(t, tempDir, clientset, emptydir.ProbeVolumePlugins()) } func TestCanSupport(t *testing.T) { @@ -1322,3 +1488,30 @@ func doTestCleanAndTeardown(plugin volume.VolumePlugin, podUID types.UID, testVo t.Errorf("TearDown() failed: %v", err) } } + +func mustMakeRoot(t *testing.T, cn string) string { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("Error while generating key: %v", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + CommonName: cn, + }, + IsCA: true, + BasicConstraintsValid: true, + } + + cert, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) + if err != nil { + t.Fatalf("Error while making certificate: %v", err) + } + + return string(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: nil, + Bytes: cert, + })) +} diff --git a/pkg/volume/testing/volume_host.go b/pkg/volume/testing/volume_host.go index e3557bb1b15..51035fb81d0 100644 --- a/pkg/volume/testing/volume_host.go +++ b/pkg/volume/testing/volume_host.go @@ -17,6 +17,7 @@ limitations under the License. package testing import ( + "bytes" "context" "fmt" "net" @@ -437,3 +438,30 @@ func (f *fakeKubeletVolumeHost) WaitForCacheSync() error { func (f *fakeKubeletVolumeHost) GetHostUtil() hostutil.HostUtils { return f.hostUtil } + +func (f *fakeKubeletVolumeHost) GetTrustAnchorsByName(name string, allowMissing bool) ([]byte, error) { + ctb, err := f.kubeClient.CertificatesV1alpha1().ClusterTrustBundles().Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("while getting ClusterTrustBundle %s: %w", name, err) + } + + return []byte(ctb.Spec.TrustBundle), nil +} + +// Note: we do none of the deduplication and sorting that the real deal should do. +func (f *fakeKubeletVolumeHost) GetTrustAnchorsBySigner(signerName string, labelSelector *metav1.LabelSelector, allowMissing bool) ([]byte, error) { + ctbList, err := f.kubeClient.CertificatesV1alpha1().ClusterTrustBundles().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("while listing all ClusterTrustBundles: %w", err) + } + + fullSet := bytes.Buffer{} + for i, ctb := range ctbList.Items { + fullSet.WriteString(ctb.Spec.TrustBundle) + if i != len(ctbList.Items)-1 { + fullSet.WriteString("\n") + } + } + + return fullSet.Bytes(), nil +} diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 6989a72ff18..fca23475120 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -258,6 +258,17 @@ func (p *Plugin) admitPodCreate(nodeName string, a admission.Attributes) error { if hasConfigMaps { return admission.NewForbidden(a, fmt.Errorf("node %q can not create pods that reference configmaps", nodeName)) } + + for _, vol := range pod.Spec.Volumes { + if vol.VolumeSource.Projected != nil { + for _, src := range vol.VolumeSource.Projected.Sources { + if src.ClusterTrustBundle != nil { + return admission.NewForbidden(a, fmt.Errorf("node %q can not create pods that reference clustertrustbundles", nodeName)) + } + } + } + } + for _, v := range pod.Spec.Volumes { if v.PersistentVolumeClaim != nil { return admission.NewForbidden(a, fmt.Errorf("node %q can not create pods that reference persistentvolumeclaims", nodeName)) diff --git a/plugin/pkg/admission/noderestriction/admission_test.go b/plugin/pkg/admission/noderestriction/admission_test.go index 2d9a0824089..8bfd0bfd0ea 100644 --- a/plugin/pkg/admission/noderestriction/admission_test.go +++ b/plugin/pkg/admission/noderestriction/admission_test.go @@ -394,6 +394,9 @@ func Test_nodePlugin_Admit(t *testing.T) { configmappod, _ := makeTestPod("ns", "myconfigmappod", "mynode", true) configmappod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{ConfigMap: &api.ConfigMapVolumeSource{LocalObjectReference: api.LocalObjectReference{Name: "foo"}}}}} + ctbpod, _ := makeTestPod("ns", "myctbpod", "mynode", true) + ctbpod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{Projected: &api.ProjectedVolumeSource{Sources: []api.VolumeProjection{{ClusterTrustBundle: &api.ClusterTrustBundleProjection{Name: pointer.String("foo")}}}}}}} + pvcpod, _ := makeTestPod("ns", "mypvcpod", "mynode", true) pvcpod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "foo"}}}} @@ -866,6 +869,12 @@ func Test_nodePlugin_Admit(t *testing.T) { attributes: admission.NewAttributesRecord(configmappod, nil, podKind, configmappod.Namespace, configmappod.Name, podResource, "", admission.Create, &metav1.CreateOptions{}, false, mynode), err: "reference configmaps", }, + { + name: "forbid create of pod referencing clustertrustbundle", + podsGetter: noExistingPods, + attributes: admission.NewAttributesRecord(ctbpod, nil, podKind, ctbpod.Namespace, ctbpod.Name, podResource, "", admission.Create, &metav1.CreateOptions{}, false, mynode), + err: "reference clustertrustbundles", + }, { name: "forbid create of pod referencing persistentvolumeclaim", podsGetter: noExistingPods, diff --git a/plugin/pkg/admission/serviceaccount/admission.go b/plugin/pkg/admission/serviceaccount/admission.go index 2d7dfee6993..c844a051c24 100644 --- a/plugin/pkg/admission/serviceaccount/admission.go +++ b/plugin/pkg/admission/serviceaccount/admission.go @@ -210,9 +210,6 @@ func (s *Plugin) Validate(ctx context.Context, a admission.Attributes, o admissi if projSource.ServiceAccountToken != nil { return admission.NewForbidden(a, fmt.Errorf("a mirror pod may not use ServiceAccountToken volume projections")) } - if projSource.ClusterTrustBundle != nil { - return admission.NewForbidden(a, fmt.Errorf("a mirror pod may not use ClusterTrustBundle volume projections")) - } } } }