From b09ca8c2c84962452fb3337f632904f3cfcb8ce2 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Fri, 10 Jan 2025 11:24:30 -0600 Subject: [PATCH 1/2] Use SAR to allow dynamic audiences for node audience restriction Signed-off-by: Anish Ramasekar --- .../admission/noderestriction/admission.go | 47 ++++++- .../noderestriction/admission_test.go | 118 +++++++++++++++++- 2 files changed, 156 insertions(+), 9 deletions(-) diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 79581bebe3f..45f790913e6 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" apiserveradmission "k8s.io/apiserver/pkg/admission/initializer" + "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/client-go/informers" corev1lister "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" @@ -84,6 +85,8 @@ type Plugin struct { pvGetter corev1lister.PersistentVolumeLister csiTranslator csitrans.CSITranslator + authz authorizer.Authorizer + expansionRecoveryEnabled bool dynamicResourceAllocationEnabled bool allowInsecureKubeletCertificateSigningRequests bool @@ -94,6 +97,7 @@ var ( _ admission.Interface = &Plugin{} _ apiserveradmission.WantsExternalKubeInformerFactory = &Plugin{} _ apiserveradmission.WantsFeatures = &Plugin{} + _ apiserveradmission.WantsAuthorizer = &Plugin{} ) // InspectFeatureGates allows setting bools without taking a dep on a global variable @@ -137,10 +141,20 @@ func (p *Plugin) ValidateInitialization() error { if p.pvGetter == nil { return fmt.Errorf("%s requires a PV getter", PluginName) } + if p.authz == nil { + return fmt.Errorf("%s requires an authorizer", PluginName) + } } return nil } +// SetAuthorizer sets the authorizer. +func (p *Plugin) SetAuthorizer(authz authorizer.Authorizer) { + if p.serviceAccountNodeAudienceRestriction { + p.authz = authz + } +} + var ( podResource = api.Resource("pods") nodeResource = api.Resource("nodes") @@ -624,7 +638,7 @@ func (p *Plugin) admitServiceAccount(ctx context.Context, nodeName string, a adm } if p.serviceAccountNodeAudienceRestriction { - if err := p.validateNodeServiceAccountAudience(ctx, tr, pod); err != nil { + if err := p.validateNodeServiceAccountAudience(ctx, tr, pod, a); err != nil { return admission.NewForbidden(a, err) } } @@ -638,7 +652,7 @@ func (p *Plugin) admitServiceAccount(ctx context.Context, nodeName string, a adm return nil } -func (p *Plugin) validateNodeServiceAccountAudience(ctx context.Context, tr *authenticationapi.TokenRequest, pod *v1.Pod) error { +func (p *Plugin) validateNodeServiceAccountAudience(ctx context.Context, tr *authenticationapi.TokenRequest, pod *v1.Pod, a admission.Attributes) error { // ensure all items in tr.Spec.Audiences are present in a volume mount in the pod requestedAudience := "" switch len(tr.Spec.Audiences) { @@ -654,10 +668,33 @@ func (p *Plugin) validateNodeServiceAccountAudience(ctx context.Context, tr *aut if err != nil { return fmt.Errorf("error validating audience %q: %w", requestedAudience, err) } - if !foundAudiencesInPodSpec { - return fmt.Errorf("audience %q not found in pod spec volume", requestedAudience) + if foundAudiencesInPodSpec { + return nil } - return nil + + userInfo := a.GetUserInfo() + attrs := authorizer.AttributesRecord{ + User: userInfo, // this is the user info of the node requesting the token + Verb: "request-serviceaccounts-token-audience", + Namespace: a.GetNamespace(), + APIGroup: "", + APIVersion: "v1", + Resource: requestedAudience, // this gives us the audience for which node is requesting a token for; wildcard will allow all audiences + Name: a.GetName(), // this gives us the service account name for which node is requesting a token for; if not set, default will allow all service accounts + ResourceRequest: true, + } + + authorized, _, err := p.authz.Authorize(ctx, attrs) + // an authorizer like RBAC could encounter evaluation errors and still allow the request, so authorizer decision is checked before error here. + // following the same pattern as withAuthorization (ref: https://github.com/kubernetes/kubernetes/blob/2b025e645975d6d51bf38c008f972c632cf49657/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go#L71-L91) + if authorized == authorizer.DecisionAllow { + return nil + } + if err != nil { + return fmt.Errorf("audience %q not found in pod spec volume, error authorizing %s to request tokens for this audience: %w", requestedAudience, userInfo.GetName(), err) + } + + return fmt.Errorf("audience %q not found in pod spec volume, %s is not authorized to request tokens for this audience", requestedAudience, userInfo.GetName()) } func (p *Plugin) podReferencesAudience(ctx context.Context, pod *v1.Pod, audience string) (bool, error) { diff --git a/plugin/pkg/admission/noderestriction/admission_test.go b/plugin/pkg/admission/noderestriction/admission_test.go index fa8b194b437..d3b3da0b2c5 100644 --- a/plugin/pkg/admission/noderestriction/admission_test.go +++ b/plugin/pkg/admission/noderestriction/admission_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/util/feature" corev1lister "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" @@ -110,6 +111,9 @@ func makeTestPodEviction(name string) *policy.Eviction { func makeTokenRequest(podname string, poduid types.UID, audiences []string) *authenticationapi.TokenRequest { tr := &authenticationapi.TokenRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + }, Spec: authenticationapi.TokenRequestSpec{ Audiences: audiences, }, @@ -225,6 +229,7 @@ type admitTestCase struct { features featuregate.FeatureGate setupFunc func(t *testing.T) err string + authz authorizer.Authorizer } func (a *admitTestCase) run(t *testing.T) { @@ -241,6 +246,7 @@ func (a *admitTestCase) run(t *testing.T) { c.csiDriverGetter = a.csiDriverGetter c.pvcGetter = a.pvcGetter c.pvGetter = a.pvGetter + c.authz = a.authz err := c.Admit(context.TODO(), a.attributes, nil) if (err == nil) != (len(a.err) == 0) { t.Errorf("nodePlugin.Admit() error = %v, expected %v", err, a.err) @@ -1311,7 +1317,14 @@ func Test_nodePlugin_Admit(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypod.Name, coremypod.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), - err: `serviceaccounts "mysa" is forbidden: audience "foo" not found in pod spec volume`, + err: `serviceaccounts "mysa" is forbidden: audience "foo" not found in pod spec volume, system:node:mynode is not authorized to request tokens for this audience`, + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "foo", + decision: authorizer.DecisionDeny, + }, }, { name: "allow create of token when audience in pod --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled", @@ -1334,7 +1347,14 @@ func Test_nodePlugin_Admit(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithCSI.Name, v1mypodWithCSI.UID, []string{"bar"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), - err: `audience "bar" not found in pod spec volume`, + err: `audience "bar" not found in pod spec volume, system:node:mynode is not authorized to request tokens for this audience`, + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "bar", + decision: authorizer.DecisionDeny, + }, }, { name: "forbid create of token when audience in pod --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled, csidriver not found", @@ -1347,6 +1367,7 @@ func Test_nodePlugin_Admit(t *testing.T) { }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithCSI.Name, v1mypodWithCSI.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), err: `error validating audience "foo": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`, + authz: fakeAuthorizer{}, }, { name: "allow create of token when audience in pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled", @@ -1373,7 +1394,14 @@ func Test_nodePlugin_Admit(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithPVCRefCSI.Name, v1mypodWithPVCRefCSI.UID, []string{"bar"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), - err: `audience "bar" not found in pod spec volume`, + err: `audience "bar" not found in pod spec volume, system:node:mynode is not authorized to request tokens for this audience`, + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "bar", + decision: authorizer.DecisionDeny, + }, }, { name: "forbid create of token when audience in pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled, pvc not found", @@ -1388,6 +1416,7 @@ func Test_nodePlugin_Admit(t *testing.T) { }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithPVCRefCSI.Name, v1mypodWithPVCRefCSI.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), err: `error validating audience "foo": persistentvolumeclaim "pvclaim" not found`, + authz: fakeAuthorizer{}, }, { name: "forbid create of token when audience in pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled, pv not found", @@ -1402,6 +1431,7 @@ func Test_nodePlugin_Admit(t *testing.T) { }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithPVCRefCSI.Name, v1mypodWithPVCRefCSI.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), err: `error validating audience "foo": persistentvolume "pvname" not found`, + authz: fakeAuthorizer{}, }, { name: "allow create of token when audience in pod --> ephemeral --> pvc --> pv --> csi --> driver --> tokenrequest with audience and ServiceAccountNodeAudienceRestriction is enabled", @@ -1428,7 +1458,14 @@ func Test_nodePlugin_Admit(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodWithEphemeralVolume.Name, v1mypodWithEphemeralVolume.UID, []string{"bar"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), - err: `audience "bar" not found in pod spec volume`, + err: `audience "bar" not found in pod spec volume, system:node:mynode is not authorized to request tokens for this audience`, + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "bar", + decision: authorizer.DecisionDeny, + }, }, { name: "allow create of token when ServiceAccountNodeAudienceRestriction is disabled, pvc not found should not be checked", @@ -1503,6 +1540,47 @@ func Test_nodePlugin_Admit(t *testing.T) { }, attributes: admission.NewAttributesRecord(makeTokenRequest(coremypodIntreeInlineVolToCSI.Name, v1mypodIntreeInlineVolToCSI.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), }, + { + name: "allow create of token when ServiceAccountNodeAudienceRestriction is enabled, clusterrole and clusterrolebinding are configured", + podsGetter: existingPods, + csiDriverGetter: noexistingCSIDriverLister, + pvcGetter: noexistingPVCLister, + pvGetter: noexistingPVLister, + features: feature.DefaultFeatureGate, + setupFunc: func(t *testing.T) { + t.Helper() + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) + }, + attributes: admission.NewAttributesRecord(makeTokenRequest(coremypod.Name, v1mypod.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "foo", + decision: authorizer.DecisionAllow, + }, + }, + { + name: "forbid create of token when ServiceAccountNodeAudienceRestriction is enabled, clusterrole and clusterrolebinding for audience not configured", + podsGetter: existingPods, + csiDriverGetter: noexistingCSIDriverLister, + pvcGetter: noexistingPVCLister, + pvGetter: noexistingPVLister, + features: feature.DefaultFeatureGate, + setupFunc: func(t *testing.T) { + t.Helper() + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ServiceAccountNodeAudienceRestriction, true) + }, + attributes: admission.NewAttributesRecord(makeTokenRequest(coremypod.Name, v1mypod.UID, []string{"foo"}), nil, tokenrequestKind, coremypod.Namespace, "mysa", svcacctResource, "token", admission.Create, &metav1.CreateOptions{}, false, mynode), + authz: fakeAuthorizer{ + t: t, + serviceAccountName: "mysa", + namespace: coremypod.Namespace, + requestAudience: "foo", + decision: authorizer.DecisionDeny, + }, + err: `serviceaccounts "mysa" is forbidden: audience "foo" not found in pod spec volume, system:node:mynode is not authorized to request tokens for this audience`, + }, // Unrelated objects { @@ -2234,3 +2312,35 @@ func checkNilError(t *testing.T, err error) { t.Fatalf("unexpected error: %v", err) } } + +type fakeAuthorizer struct { + t *testing.T + serviceAccountName string + namespace string + requestAudience string + decision authorizer.Decision + err error +} + +func (f fakeAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) { + if f.err != nil { + return f.decision, "forced error", f.err + } + + expectedAttrs := authorizer.AttributesRecord{ + User: &user.DefaultInfo{Name: "system:node:mynode", Groups: []string{"system:nodes"}}, + Verb: "request-serviceaccounts-token-audience", + Namespace: f.namespace, + APIGroup: "", + APIVersion: "v1", + Resource: f.requestAudience, + Name: f.serviceAccountName, + ResourceRequest: true, + } + + if !reflect.DeepEqual(a, expectedAttrs) { + f.t.Errorf("expected attributes: %v, got: %v", expectedAttrs, a) + } + + return f.decision, "", nil +} From 3f5d30543d857f9546f3f7545cba3a4a6c9b5986 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Thu, 27 Feb 2025 13:52:51 -0600 Subject: [PATCH 2/2] Add integration tests for using SAR with node audience restriction Signed-off-by: Anish Ramasekar --- test/integration/auth/node_test.go | 464 +++++++++++++++++++++++++++-- 1 file changed, 432 insertions(+), 32 deletions(-) diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index a3c4433f5c3..a75bec98e3c 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" @@ -799,26 +800,26 @@ func TestNodeRestrictionServiceAccount(t *testing.T) { }) t.Run("service account token request is forbidden when pod is not found", func(t *testing.T) { - expectNotFound(t, createTokenRequest(node1Client, "uid1", "")) + expectNotFound(t, createTokenRequest(node1Client, "uid1", tokenRequestWithAudiences(""))) }) t.Run("uid in token request does not match pod uid is forbidden", func(t *testing.T) { createPod(t, superuserClient, nil) - expectedForbiddenMessage(t, createTokenRequest(node1Client, "random-uid", ""), "the UID in the bound object reference (random-uid) does not match the UID in record. The object might have been deleted and then recreated") + expectedForbiddenMessage(t, createTokenRequest(node1Client, "random-uid", tokenRequestWithAudiences("")), "the UID in the bound object reference (random-uid) does not match the UID in record. The object might have been deleted and then recreated") deletePod(t, superuserClient, "pod1") }) t.Run("node requesting token for pod bound to different node is forbidden", func(t *testing.T) { pod := createPod(t, superuserClient, nil) - expectedForbiddenMessage(t, createTokenRequest(node2Client, pod.UID, ""), "node requested token bound to a pod scheduled on a different node") + expectedForbiddenMessage(t, createTokenRequest(node2Client, pod.UID, tokenRequestWithAudiences("")), "node requested token bound to a pod scheduled on a different node") deletePod(t, superuserClient, "pod1") }) t.Run("service account token request is successful", func(t *testing.T) { // create a pod as an admin to add object references pod := createPod(t, superuserClient, nil) - createDefaultServiceAccount(t, superuserClient) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "")) + createServiceAccount(t, superuserClient, "ns", "default") + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(""))) }) } @@ -830,6 +831,15 @@ func TestNodeRestrictionServiceAccount(t *testing.T) { // 3. pod --> csi --> driver --> tokenrequest with audience // 4. pod --> projected --> service account token with audience // +// - allows kubelet to request a token for a service account based on subjectacccessreview +// 1. clusterrole and clusterrolebinding allowing a node to request specific audience for a service account +// 2. clusterrole and rolebinding allowing a node to request specific audience for a service account +// 3. role and rolebinding allowing a node to request specific audience for a service account in a namespace +// 4. clusterrole and rolebinding allowing a node to request specific audience for any service account in a namespace +// 5. clusterrole and rolebinding allowing any node to request any audience for a service account in a namespace +// 6. clusterrole and rolebinding allowing any node to request any audience for any service account in a namespace +// 7. api server audience "" configured with rbac role +// // - forbids kubelet to request a token for a service account that's not in the pod spec // when the ServiceAccountNodeAudienceRestriction feature is enabled. func TestNodeRestrictionServiceAccountAudience(t *testing.T) { @@ -931,26 +941,26 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { node1Client, _ := clientsetForToken(tokenNode1, clientConfig) createNode(t, node1Client, "node1") - createDefaultServiceAccount(t, superuserClient) + createServiceAccount(t, superuserClient, "ns", "default") t.Run("projected volume source with empty audience works", func(t *testing.T) { projectedVolumeSourceEmptyAudience := &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: "", Path: "path"}}}} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Projected: projectedVolumeSourceEmptyAudience}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(""))) deletePod(t, superuserClient, "pod1") }) t.Run("projected volume source with non-empty audience works", func(t *testing.T) { projectedVolumeSource := &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: "projected-audience", Path: "path"}}}} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Projected: projectedVolumeSource}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "projected-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("projected-audience"))) deletePod(t, superuserClient, "pod1") }) t.Run("pod --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) { csiDriverVolumeSource := &corev1.CSIVolumeSource{Driver: "com.example.csi.mydriver"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{CSI: csiDriverVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidrivernotfound-audience"), `error validating audience "csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidrivernotfound-audience")), `error validating audience "csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) deletePod(t, superuserClient, "pod1") }) @@ -958,7 +968,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") csiDriverVolumeSource := &corev1.CSIVolumeSource{Driver: "com.example.csi.mydriver"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{CSI: csiDriverVolumeSource}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience"))) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -966,7 +976,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) { persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "pvc-csidrivernotfound-audience"), `error validating audience "pvc-csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("pvc-csidrivernotfound-audience")), `error validating audience "pvc-csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) deletePod(t, superuserClient, "pod1") }) @@ -974,7 +984,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { createCSIDriver(t, superuserClient, "pvcnotfound-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc1"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "pvcnotfound-audience"), `error validating audience "pvcnotfound-audience": persistentvolumeclaim "mypvc1" not found`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("pvcnotfound-audience")), `error validating audience "pvcnotfound-audience": persistentvolumeclaim "mypvc1" not found`) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -983,7 +993,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { createCSIDriver(t, superuserClient, "pvccsidriver-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "pvccsidriver-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("pvccsidriver-audience"))) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -996,7 +1006,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { VolumeName: "mypv", }}} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "ephemeral-csidrivernotfound-audience"), `error validating audience "ephemeral-csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("ephemeral-csidrivernotfound-audience")), `error validating audience "ephemeral-csidrivernotfound-audience": csidriver.storage.k8s.io "com.example.csi.mydriver" not found`) deletePod(t, superuserClient, "pod1") }) @@ -1009,7 +1019,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { VolumeName: "mypv", }}} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "ephemeralcsidriver-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("ephemeralcsidriver-audience"))) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -1017,7 +1027,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { t.Run("csidriver exists but tokenrequest audience not found should be forbidden", func(t *testing.T) { createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") pod := createPod(t, superuserClient, nil) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience-not-found")), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -1026,7 +1036,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience-not-found")), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -1040,7 +1050,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { VolumeName: "mypv", }}} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}}) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience-not-found")), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) @@ -1048,7 +1058,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { t.Run("intree pv to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com") pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-azurefile"}}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience"))) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "file.csi.azure.com") }) @@ -1056,14 +1066,288 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { t.Run("intree inline volume to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com") pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{ShareName: "default", SecretName: "mypvsecret"}}}}) - expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("csidriver-audience"))) deletePod(t, superuserClient, "pod1") deleteCSIDriver(t, superuserClient, "file.csi.azure.com") }) t.Run("token request with multiple audiences should be forbidden", func(t *testing.T) { pod := createPod(t, superuserClient, nil) - expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "audience1", "audience2"), "node may only request 0 or 1 audiences") + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1", "audience2")), "node may only request 0 or 1 audiences") + deletePod(t, superuserClient, "pod1") + }) + + t.Run("clusterrole and clusterrolebinding allowing node1 to request audience1 should work", func(t *testing.T) { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"audience1"}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{"some-random-name"}, + }}, + } + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "system:node:node1"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "audience-access-for-node1"}, + } + + createServiceAccount(t, superuserClient, "ns", "some-random-name") + pod := createPod(t, superuserClient, nil, podWithServiceAccountName("some-random-name")) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("some-random-name")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"), tokenRequestWithName("some-random-name")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACClusterRole(t, cr, superuserClient) + createRBACClusterRoleBinding(t, crb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("some-random-name"))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"), tokenRequestWithName("some-random-name")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACClusterRole(t, cr, superuserClient) + deleteRBACClusterRoleBinding(t, crb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("some-random-name")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + deleteServiceAccount(t, superuserClient, "ns", "some-random-name") + }) + + t.Run("clusterrole and rolebinding in a namespace allowing node1 to request audience1 should work", func(t *testing.T) { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"audience1"}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{"default"}, + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "system:node:node1"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "audience-access-for-node1"}, + } + + pod := createPod(t, superuserClient, nil) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACClusterRole(t, cr, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACClusterRole(t, cr, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + }) + + t.Run("role and rolebinding in a namespace allowing node1 to request audience1 should work", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"audience1"}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{"default"}, + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "system:node:node1"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-node1"}, + } + + pod := createPod(t, superuserClient, nil) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + }) + + t.Run("clusterrole and rolebinding in a namespace allowing node1 to request audience1, wildcard for service account names", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"audience1"}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{}, // resource name empty means all service accounts + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "system:node:node1"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-node1"}, + } + + createServiceAccount(t, superuserClient, "ns", "custom-sa") + pod := createPod(t, superuserClient, nil, podWithServiceAccountName("custom-sa")) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("custom-sa")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"), tokenRequestWithName("custom-sa")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("custom-sa"))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"), tokenRequestWithName("custom-sa")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"), tokenRequestWithName("custom-sa")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + deleteServiceAccount(t, superuserClient, "ns", "custom-sa") + }) + + t.Run("clusterrole and rolebinding in a namespace allowing nodes to request any audience, wildcard for audiences", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-nodes", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"*"}, // wildcard for audiences + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{"default"}, + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "Group", Name: "system:nodes"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-nodes"}, + } + + pod := createPod(t, superuserClient, nil) + + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + randomAudience := rand.String(10) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(randomAudience)), `audience "`+randomAudience+`" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"))) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"))) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(""))) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(randomAudience))) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + }) + + t.Run("clusterrole and rolebinding in a namespace allowing nodes to request any audience, any service account, wildcard for audiences", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-nodes", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"*"}, // wildcard for audiences + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{}, // empty resource name means all service accounts + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "Group", Name: "system:nodes"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-nodes"}, + } + + pod := createPod(t, superuserClient, nil) + + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2")), `audience "audience2" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1"))) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience2"))) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + }) + + t.Run("api server audience configured with rbac role", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-nodes", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{""}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{}, // empty resource name means all service accounts + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "Group", Name: "system:nodes"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-nodes"}, + } + + pod := createPod(t, superuserClient, nil, podWithAutoMountServiceAccountToken(false)) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("")), `audience "" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences(""))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("")), `audience "" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + deletePod(t, superuserClient, "pod1") + }) + + t.Run("audience with : and '/' configured with rbac role", func(t *testing.T) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-nodes", Namespace: "ns"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"myaud://audience1/audience2.com"}, + Verbs: []string{"request-serviceaccounts-token-audience"}, + ResourceNames: []string{}, // empty resource name means all service accounts + }}, + } + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "audience-access-for-node1", Namespace: "ns"}, + Subjects: []rbacv1.Subject{{Kind: "Group", Name: "system:nodes"}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "audience-access-for-nodes"}, + } + + pod := createPod(t, superuserClient, nil, podWithAutoMountServiceAccountToken(false)) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("myaud://audience1/audience2.com")), `audience "myaud://audience1/audience2.com" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + createRBACRole(t, role, superuserClient) + createRBACRoleBinding(t, rb, superuserClient) + + expectAllowed(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("myaud://audience1/audience2.com"))) + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("audience1")), `audience "audience1" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) + + deleteRBACRole(t, role, superuserClient) + deleteRBACRoleBinding(t, rb, superuserClient) + // After the delete use a expectedForbiddenMessage to wait for the RBAC authorizer to catch up. + expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, tokenRequestWithAudiences("myaud://audience1/audience2.com")), `audience "myaud://audience1/audience2.com" not found in pod spec volume, system:node:node1 is not authorized to request tokens for this audience`) deletePod(t, superuserClient, "pod1") }) } @@ -1080,17 +1364,47 @@ func createKubeConfigFileForRestConfig(t *testing.T, restConfig *rest.Config) st return kubeConfigFile } -func createPod(t *testing.T, client clientset.Interface, volumes []corev1.Volume) *corev1.Pod { +type podOptions struct { + serviceAccountName string + autoMountServiceAccountToken *bool +} + +type podOption func(*podOptions) + +func podWithServiceAccountName(name string) podOption { + return func(opts *podOptions) { + opts.serviceAccountName = name + } +} + +func podWithAutoMountServiceAccountToken(autoMount bool) podOption { + return func(opts *podOptions) { + opts.autoMountServiceAccountToken = ptr.To(autoMount) + } +} + +func createPod(t *testing.T, client clientset.Interface, volumes []corev1.Volume, opts ...podOption) *corev1.Pod { t.Helper() + + options := &podOptions{ + serviceAccountName: "default", + autoMountServiceAccountToken: ptr.To(true), + } + + for _, opt := range opts { + opt(options) + } + pod, err := client.CoreV1().Pods("ns").Create(context.TODO(), &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", }, Spec: corev1.PodSpec{ - NodeName: "node1", - Containers: []corev1.Container{{Name: "image", Image: "busybox"}}, - ServiceAccountName: "default", - Volumes: volumes, + NodeName: "node1", + Containers: []corev1.Container{{Name: "image", Image: "busybox"}}, + ServiceAccountName: options.serviceAccountName, + Volumes: volumes, + AutomountServiceAccountToken: options.autoMountServiceAccountToken, }, }, metav1.CreateOptions{}) checkNilError(t, err) @@ -1110,9 +1424,37 @@ func createNode(t *testing.T, client clientset.Interface, nodeName string) { checkNilError(t, err) } -func createTokenRequest(client clientset.Interface, uid types.UID, audiences ...string) func() error { +type tokenRequestOptions struct { + name string + audiences []string +} + +type tokenRequestOption func(*tokenRequestOptions) + +func tokenRequestWithName(name string) tokenRequestOption { + return func(opts *tokenRequestOptions) { + opts.name = name + } +} + +func tokenRequestWithAudiences(audiences ...string) tokenRequestOption { + return func(opts *tokenRequestOptions) { + opts.audiences = audiences + } +} + +func createTokenRequest(client clientset.Interface, uid types.UID, opts ...tokenRequestOption) func() error { + options := &tokenRequestOptions{ + name: "default", + audiences: []string{}, + } + + for _, opt := range opts { + opt(options) + } + return func() error { - _, err := client.CoreV1().ServiceAccounts("ns").CreateToken(context.TODO(), "default", &authenticationv1.TokenRequest{ + _, err := client.CoreV1().ServiceAccounts("ns").CreateToken(context.TODO(), options.name, &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ BoundObjectRef: &authenticationv1.BoundObjectReference{ Kind: "Pod", @@ -1120,7 +1462,7 @@ func createTokenRequest(client clientset.Interface, uid types.UID, audiences ... APIVersion: "v1", UID: uid, }, - Audiences: audiences, + Audiences: options.audiences, }, }, metav1.CreateOptions{}) return err @@ -1145,15 +1487,21 @@ func deleteCSIDriver(t *testing.T, client clientset.Interface, driverName string checkNilError(t, client.StorageV1().CSIDrivers().Delete(context.TODO(), driverName, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)})) } -func createDefaultServiceAccount(t *testing.T, client clientset.Interface) { +func createServiceAccount(t *testing.T, client clientset.Interface, namespace, name string) { t.Helper() - _, err := client.CoreV1().ServiceAccounts("ns").Create(context.TODO(), &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, + _, err := client.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name}, }, metav1.CreateOptions{}) checkNilError(t, err) } +func deleteServiceAccount(t *testing.T, client clientset.Interface, namespace, name string) { + t.Helper() + + checkNilError(t, client.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)})) +} + func configureRBACForServiceAccountToken(t *testing.T, client clientset.Interface) { t.Helper() @@ -1182,3 +1530,55 @@ func configureRBACForServiceAccountToken(t *testing.T, client clientset.Interfac }, metav1.UpdateOptions{}) checkNilError(t, err) } + +func createRBACClusterRole(t *testing.T, clusterrole *rbacv1.ClusterRole, client clientset.Interface) { + t.Helper() + + _, err := client.RbacV1().ClusterRoles().Create(context.TODO(), clusterrole, metav1.CreateOptions{}) + checkNilError(t, err) +} + +func createRBACClusterRoleBinding(t *testing.T, clusterrolebinding *rbacv1.ClusterRoleBinding, client clientset.Interface) { + t.Helper() + + _, err := client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterrolebinding, metav1.CreateOptions{}) + checkNilError(t, err) +} + +func createRBACRole(t *testing.T, role *rbacv1.Role, client clientset.Interface) { + t.Helper() + + _, err := client.RbacV1().Roles(role.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}) + checkNilError(t, err) +} + +func createRBACRoleBinding(t *testing.T, rolebinding *rbacv1.RoleBinding, client clientset.Interface) { + t.Helper() + + _, err := client.RbacV1().RoleBindings(rolebinding.Namespace).Create(context.TODO(), rolebinding, metav1.CreateOptions{}) + checkNilError(t, err) +} + +func deleteRBACClusterRole(t *testing.T, clusterrole *rbacv1.ClusterRole, client clientset.Interface) { + t.Helper() + + checkNilError(t, client.RbacV1().ClusterRoles().Delete(context.TODO(), clusterrole.Name, metav1.DeleteOptions{})) +} + +func deleteRBACClusterRoleBinding(t *testing.T, clusterrolebinding *rbacv1.ClusterRoleBinding, client clientset.Interface) { + t.Helper() + + checkNilError(t, client.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterrolebinding.Name, metav1.DeleteOptions{})) +} + +func deleteRBACRole(t *testing.T, role *rbacv1.Role, client clientset.Interface) { + t.Helper() + + checkNilError(t, client.RbacV1().Roles(role.Namespace).Delete(context.TODO(), role.Name, metav1.DeleteOptions{})) +} + +func deleteRBACRoleBinding(t *testing.T, rolebinding *rbacv1.RoleBinding, client clientset.Interface) { + t.Helper() + + checkNilError(t, client.RbacV1().RoleBindings(rolebinding.Namespace).Delete(context.TODO(), rolebinding.Name, metav1.DeleteOptions{})) +}