From 62809dd0de900c9fb466446799857185349e02b4 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Wed, 5 Feb 2025 14:06:04 -0600 Subject: [PATCH] node audience restriction: use csi translator to convert intree inline_vol/pv to csi Signed-off-by: Anish Ramasekar --- .../admission/noderestriction/admission.go | 43 ++++++++--- test/integration/auth/node_test.go | 73 ++++++++++++++----- 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 6a69b691605..79581bebe3f 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -39,6 +39,8 @@ import ( storagelisters "k8s.io/client-go/listers/storage/v1" "k8s.io/component-base/featuregate" "k8s.io/component-helpers/storage/ephemeral" + csitrans "k8s.io/csi-translation-lib" + "k8s.io/klog/v2" kubeletapis "k8s.io/kubelet/pkg/apis" podutil "k8s.io/kubernetes/pkg/api/pod" authenticationapi "k8s.io/kubernetes/pkg/apis/authentication" @@ -80,6 +82,7 @@ type Plugin struct { csiDriverGetter storagelisters.CSIDriverLister pvcGetter corev1lister.PersistentVolumeClaimLister pvGetter corev1lister.PersistentVolumeLister + csiTranslator csitrans.CSITranslator expansionRecoveryEnabled bool dynamicResourceAllocationEnabled bool @@ -109,6 +112,7 @@ func (p *Plugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactor p.csiDriverGetter = f.Storage().V1().CSIDrivers().Lister() p.pvcGetter = f.Core().V1().PersistentVolumeClaims().Lister() p.pvGetter = f.Core().V1().PersistentVolumes().Lister() + p.csiTranslator = csitrans.New() } } @@ -189,7 +193,7 @@ func (p *Plugin) Admit(ctx context.Context, a admission.Attributes, o admission. } case svcacctResource: - return p.admitServiceAccount(nodeName, a) + return p.admitServiceAccount(ctx, nodeName, a) case leaseResource: return p.admitLease(nodeName, a) @@ -581,7 +585,7 @@ func (p *Plugin) getForbiddenLabels(modifiedLabels sets.String) sets.String { return forbiddenLabels } -func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) error { +func (p *Plugin) admitServiceAccount(ctx context.Context, nodeName string, a admission.Attributes) error { if a.GetOperation() != admission.Create { return nil } @@ -620,7 +624,7 @@ func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) er } if p.serviceAccountNodeAudienceRestriction { - if err := p.validateNodeServiceAccountAudience(tr, pod); err != nil { + if err := p.validateNodeServiceAccountAudience(ctx, tr, pod); err != nil { return admission.NewForbidden(a, err) } } @@ -634,7 +638,7 @@ func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) er return nil } -func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenRequest, pod *v1.Pod) error { +func (p *Plugin) validateNodeServiceAccountAudience(ctx context.Context, tr *authenticationapi.TokenRequest, pod *v1.Pod) error { // ensure all items in tr.Spec.Audiences are present in a volume mount in the pod requestedAudience := "" switch len(tr.Spec.Audiences) { @@ -646,7 +650,7 @@ func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenR return fmt.Errorf("node may only request 0 or 1 audiences") } - foundAudiencesInPodSpec, err := p.podReferencesAudience(pod, requestedAudience) + foundAudiencesInPodSpec, err := p.podReferencesAudience(ctx, pod, requestedAudience) if err != nil { return fmt.Errorf("error validating audience %q: %w", requestedAudience, err) } @@ -656,7 +660,7 @@ func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenR return nil } -func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, error) { +func (p *Plugin) podReferencesAudience(ctx context.Context, pod *v1.Pod, audience string) (bool, error) { var errs []error for _, v := range pod.Spec.Volumes { @@ -677,11 +681,20 @@ func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, erro switch { case v.Ephemeral != nil && v.Ephemeral.VolumeClaimTemplate != nil: pvcName := ephemeral.VolumeClaimName(pod, &v) - driverName, err = p.getCSIFromPVC(pod.Namespace, pvcName) + driverName, err = p.getCSIFromPVC(ctx, pod.Namespace, pvcName) case v.PersistentVolumeClaim != nil: - driverName, err = p.getCSIFromPVC(pod.Namespace, v.PersistentVolumeClaim.ClaimName) + driverName, err = p.getCSIFromPVC(ctx, pod.Namespace, v.PersistentVolumeClaim.ClaimName) case v.CSI != nil: driverName = v.CSI.Driver + case p.csiTranslator.IsInlineMigratable(&v): + pv, translateErr := p.csiTranslator.TranslateInTreeInlineVolumeToCSI(klog.FromContext(ctx), &v, pod.Namespace) + if translateErr != nil { + err = translateErr + break + } + if pv != nil && pv.Spec.CSI != nil { + driverName = pv.Spec.CSI.Driver + } } if err != nil { @@ -705,7 +718,7 @@ func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, erro } // getCSIFromPVC returns the CSI driver name from the PVC->PV->CSI->Driver chain -func (p *Plugin) getCSIFromPVC(namespace, claimName string) (string, error) { +func (p *Plugin) getCSIFromPVC(ctx context.Context, namespace, claimName string) (string, error) { pvc, err := p.pvcGetter.PersistentVolumeClaims(namespace).Get(claimName) if err != nil { return "", err @@ -717,6 +730,18 @@ func (p *Plugin) getCSIFromPVC(namespace, claimName string) (string, error) { if pv.Spec.CSI != nil { return pv.Spec.CSI.Driver, nil } + + if p.csiTranslator.IsPVMigratable(pv) { + // For in-tree PV, we need to convert ("translate") the PV to CSI before checking the driver name. + translatedPV, err := p.csiTranslator.TranslateInTreePVToCSI(klog.FromContext(ctx), pv) + if err != nil { + return "", err + } + if translatedPV != nil && translatedPV.Spec.CSI != nil { + return translatedPV.Spec.CSI.Driver, nil + } + } + return "", nil } diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 375d61b7b0d..a3c4433f5c3 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -908,6 +908,27 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { }, metav1.CreateOptions{}) checkNilError(t, err) + _, err = superuserClient.CoreV1().PersistentVolumeClaims("ns").Create(context.TODO(), &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "mypvc-azurefile"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1")}}, + VolumeName: "mypv-azurefile", + }, + }, metav1.CreateOptions{}) + checkNilError(t, err) + + _, err = superuserClient.CoreV1().PersistentVolumes().Create(context.TODO(), &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "mypv-azurefile"}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1")}, + ClaimRef: &corev1.ObjectReference{Namespace: "ns", Name: "mypvc-azurefile"}, + PersistentVolumeSource: corev1.PersistentVolumeSource{AzureFile: &corev1.AzureFilePersistentVolumeSource{ShareName: "share", SecretName: "secret"}}, + }, + }, metav1.CreateOptions{}) + checkNilError(t, err) + node1Client, _ := clientsetForToken(tokenNode1, clientConfig) createNode(t, node1Client, "node1") createDefaultServiceAccount(t, superuserClient) @@ -934,12 +955,12 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { }) t.Run("pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { - createCSIDriver(t, superuserClient, "csidriver-audience") + createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") csiDriverVolumeSource := &corev1.CSIVolumeSource{Driver: "com.example.csi.mydriver"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{CSI: csiDriverVolumeSource}}}) expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) { @@ -950,21 +971,21 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { }) t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - pvc not found", func(t *testing.T) { - createCSIDriver(t, superuserClient, "pvcnotfound-audience") + createCSIDriver(t, superuserClient, "pvcnotfound-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc1"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "pvcnotfound-audience"), `error validating audience "pvcnotfound-audience": persistentvolumeclaim "mypvc1" not found`) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { - createCSIDriver(t, superuserClient, "pvccsidriver-audience") + createCSIDriver(t, superuserClient, "pvccsidriver-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) expectAllowed(t, createTokenRequest(node1Client, pod.UID, "pvccsidriver-audience")) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("pod --> ephemeral --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) { @@ -980,7 +1001,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { }) t.Run("pod --> ephemeral --> pvc --> pv --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { - createCSIDriver(t, superuserClient, "ephemeralcsidriver-audience") + createCSIDriver(t, superuserClient, "ephemeralcsidriver-audience", "com.example.csi.mydriver") ephemeralVolumeSource := &corev1.EphemeralVolumeSource{VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, @@ -990,28 +1011,28 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}}) expectAllowed(t, createTokenRequest(node1Client, pod.UID, "ephemeralcsidriver-audience")) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("csidriver exists but tokenrequest audience not found should be forbidden", func(t *testing.T) { - createCSIDriver(t, superuserClient, "csidriver-audience") + createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") pod := createPod(t, superuserClient, nil) expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("pvc and csidriver exists but tokenrequest audience not found should be forbidden", func(t *testing.T) { - createCSIDriver(t, superuserClient, "csidriver-audience") + createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"} pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}}) expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") }) t.Run("ephemeral volume source with audience not found should be forbidden", func(t *testing.T) { - createCSIDriver(t, superuserClient, "csidriver-audience") + createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver") ephemeralVolumeSource := &corev1.EphemeralVolumeSource{VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, @@ -1021,7 +1042,23 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) { pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}}) expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`) deletePod(t, superuserClient, "pod1") - deleteCSIDriver(t, superuserClient) + deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver") + }) + + t.Run("intree pv to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { + createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com") + pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-azurefile"}}}}) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) + deletePod(t, superuserClient, "pod1") + deleteCSIDriver(t, superuserClient, "file.csi.azure.com") + }) + + t.Run("intree inline volume to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) { + createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com") + pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{ShareName: "default", SecretName: "mypvsecret"}}}}) + expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience")) + deletePod(t, superuserClient, "pod1") + deleteCSIDriver(t, superuserClient, "file.csi.azure.com") }) t.Run("token request with multiple audiences should be forbidden", func(t *testing.T) { @@ -1090,11 +1127,11 @@ func createTokenRequest(client clientset.Interface, uid types.UID, audiences ... } } -func createCSIDriver(t *testing.T, client clientset.Interface, audience string) { +func createCSIDriver(t *testing.T, client clientset.Interface, audience, driverName string) { t.Helper() _, err := client.StorageV1().CSIDrivers().Create(context.TODO(), &storagev1.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{Name: "com.example.csi.mydriver"}, + ObjectMeta: metav1.ObjectMeta{Name: driverName}, Spec: storagev1.CSIDriverSpec{ TokenRequests: []storagev1.TokenRequest{{Audience: audience}}, }, @@ -1102,10 +1139,10 @@ func createCSIDriver(t *testing.T, client clientset.Interface, audience string) checkNilError(t, err) } -func deleteCSIDriver(t *testing.T, client clientset.Interface) { +func deleteCSIDriver(t *testing.T, client clientset.Interface, driverName string) { t.Helper() - checkNilError(t, client.StorageV1().CSIDrivers().Delete(context.TODO(), "com.example.csi.mydriver", metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)})) + checkNilError(t, client.StorageV1().CSIDrivers().Delete(context.TODO(), driverName, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)})) } func createDefaultServiceAccount(t *testing.T, client clientset.Interface) {