mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
Merge pull request #134947 from aojea/dra_status_check
Fine-grained Authorization for ResourceClaim Status Updates
This commit is contained in:
Kubernetes Prow Robot
GitHub
@@ -65,6 +65,7 @@
|
||||
| DRAPartitionableDevices | :ballot_box_with_check: 1.36+ | | 1.33–1.35 | 1.36– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAPartitionableDevices%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAPartitionableDevices%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRAPrioritizedList | :ballot_box_with_check: 1.34+ | | 1.33 | 1.34–1.35 | 1.36– | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAPrioritizedList%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAPrioritizedList%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRAResourceClaimDeviceStatus | :ballot_box_with_check: 1.33+ | | 1.32 | 1.33– | | | | [code](https://cs.k8s.io/?q=%5CbDRAResourceClaimDeviceStatus%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAResourceClaimDeviceStatus%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRAResourceClaimGranularStatusAuthorization | :ballot_box_with_check: 1.36+ | | | 1.36– | | | DRAResourceClaimDeviceStatus<br>DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAResourceClaimGranularStatusAuthorization%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAResourceClaimGranularStatusAuthorization%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRAResourcePoolStatus | | | 1.36– | | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAResourcePoolStatus%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAResourcePoolStatus%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRASchedulerFilterTimeout | :ballot_box_with_check: 1.34+ | | | 1.34– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRASchedulerFilterTimeout%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRASchedulerFilterTimeout%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
| DRAWorkloadResourceClaims | | | 1.36– | | | | DynamicResourceAllocation<br>GenericWorkload | [code](https://cs.k8s.io/?q=%5CbDRAWorkloadResourceClaims%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAWorkloadResourceClaims%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
|
||||
|
||||
@@ -639,6 +639,12 @@
|
||||
lockToDefault: false
|
||||
preRelease: Beta
|
||||
version: "1.33"
|
||||
- name: DRAResourceClaimGranularStatusAuthorization
|
||||
versionedSpecs:
|
||||
- default: true
|
||||
lockToDefault: false
|
||||
preRelease: Beta
|
||||
version: "1.36"
|
||||
- name: DRAResourcePoolStatus
|
||||
versionedSpecs:
|
||||
- default: false
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
resourceapi "k8s.io/api/resource/v1"
|
||||
resourcealphaapi "k8s.io/api/resource/v1alpha3"
|
||||
resourcev1beta1 "k8s.io/api/resource/v1beta1"
|
||||
@@ -44,6 +45,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/kubelet/events"
|
||||
@@ -3112,7 +3114,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
||||
}).Should(gomega.MatchError(gomega.ContainSubstring("exceeded quota: object-count, requested: count/resourceclaims.resource.k8s.io=1, used: count/resourceclaims.resource.k8s.io=1, limited: count/resourceclaims.resource.k8s.io=1")), "creating second claim not allowed")
|
||||
})
|
||||
|
||||
f.It("must be possible for the driver to update the ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) {
|
||||
f.It("must be impossible for a node ServiceAccount to update the non-node ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) {
|
||||
tCtx := f.TContext(ctx)
|
||||
claim := b.ExternalClaim()
|
||||
pod := b.PodExternal(claim.Name)
|
||||
@@ -3154,17 +3156,163 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
||||
if !ok {
|
||||
framework.Failf("pod got scheduled to node %s without a plugin", scheduledPod.Spec.NodeName)
|
||||
}
|
||||
updatedResourceClaim, err := plugin.UpdateStatus(ctx, allocatedResourceClaim)
|
||||
_, err = plugin.UpdateStatus(ctx, allocatedResourceClaim)
|
||||
gomega.Expect(apierrors.IsInvalid(err)).To(gomega.BeTrueBecause("expected invalid error: %v", err))
|
||||
gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot arbitrary-node:update"))
|
||||
})
|
||||
|
||||
f.It("must be possible for a control-plane ServiceAccount to update the ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) {
|
||||
tCtx := f.TContext(ctx)
|
||||
claim := b.ExternalClaim()
|
||||
pod := b.PodExternal(claim.Name)
|
||||
b.Create(tCtx, claim, pod)
|
||||
|
||||
// Waits for the ResourceClaim to be allocated and the pod to be scheduled.
|
||||
b.TestPod(tCtx, pod)
|
||||
|
||||
ginkgo.By("Creating a ServiceAccount and Role to update ResourceClaim status")
|
||||
saName := "claim-status-updater-sa"
|
||||
roleName := "claim-status-updater-role"
|
||||
bindingName := "claim-status-updater-binding"
|
||||
|
||||
sa := &v1.ServiceAccount{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: saName, Namespace: f.Namespace.Name},
|
||||
}
|
||||
_, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, sa, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Create ClusterRole with 'update' permission on 'resourceclaims/status'
|
||||
role := &rbacv1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: roleName, Namespace: f.Namespace.Name},
|
||||
Rules: []rbacv1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{resourceapi.SchemeGroupVersion.Group},
|
||||
Resources: []string{"resourceclaims/status"},
|
||||
Verbs: []string{"update", "patch"},
|
||||
},
|
||||
{
|
||||
// Also need 'get' to fetch the claim for the update operation
|
||||
APIGroups: []string{resourceapi.SchemeGroupVersion.Group},
|
||||
Resources: []string{"resourceclaims"},
|
||||
Verbs: []string{"get"},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = f.ClientSet.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
err := f.ClientSet.RbacV1().ClusterRoles().Delete(ctx, roleName, metav1.DeleteOptions{})
|
||||
if !apierrors.IsNotFound(err) {
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
})
|
||||
// Create ClusterRoleBinding
|
||||
binding := &rbacv1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: bindingName, Namespace: f.Namespace.Name},
|
||||
Subjects: []rbacv1.Subject{
|
||||
{
|
||||
Kind: "ServiceAccount",
|
||||
Name: saName,
|
||||
Namespace: f.Namespace.Name,
|
||||
},
|
||||
},
|
||||
RoleRef: rbacv1.RoleRef{
|
||||
Kind: "ClusterRole",
|
||||
Name: roleName,
|
||||
APIGroup: rbacv1.GroupName,
|
||||
},
|
||||
}
|
||||
_, err = f.ClientSet.RbacV1().ClusterRoleBindings().Create(ctx, binding, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
err := f.ClientSet.RbacV1().ClusterRoleBindings().Delete(ctx, bindingName, metav1.DeleteOptions{})
|
||||
if !apierrors.IsNotFound(err) {
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
})
|
||||
// Create a new clientset impersonating the ServiceAccount
|
||||
saClientConfig := rest.CopyConfig(f.ClientConfig())
|
||||
saClientConfig.Impersonate = rest.ImpersonationConfig{
|
||||
UserName: fmt.Sprintf("system:serviceaccount:%s:%s", f.Namespace.Name, saName),
|
||||
}
|
||||
saClient, err := kubernetes.NewForConfig(saClientConfig)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Get the allocated claim using the admin client
|
||||
allocatedResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(allocatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation.Devices.Results).To(gomega.HaveLen(1))
|
||||
|
||||
ginkgo.By("Setting the device status a first time (via ServiceAccount without proper RBAC) for driver " + b.DriverName())
|
||||
allocatedResourceClaim.Status.Devices = append(allocatedResourceClaim.Status.Devices,
|
||||
resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
Conditions: []metav1.Condition{{Type: "a", Status: "True", Message: "c", Reason: "d", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: &runtime.RawExtension{Raw: []byte(`{"foo":"bar"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
InterfaceName: "inf1",
|
||||
IPs: []string{"10.9.8.0/24", "2001:db8::/64"},
|
||||
HardwareAddress: "bc:1c:b6:3e:b8:25",
|
||||
},
|
||||
})
|
||||
|
||||
// Update the ResourceClaim status using the impersonated client
|
||||
// Wait for the initial ClusterRoleBinding to propagate AND for the DRA validation to explicitly reject it.
|
||||
gomega.Eventually(ctx, func(ctx context.Context) error {
|
||||
_, err := saClient.ResourceV1().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, allocatedResourceClaim, metav1.UpdateOptions{})
|
||||
if err == nil {
|
||||
return fmt.Errorf("expected request to be rejected by DRA validation, but it succeeded")
|
||||
}
|
||||
// If we get a pure Forbidden (403), the basic RBAC cache hasn't updated yet. Trigger a retry.
|
||||
if apierrors.IsForbidden(err) {
|
||||
return fmt.Errorf("standard RBAC cache not synced yet, got pure 403 Forbidden")
|
||||
}
|
||||
if apierrors.IsInvalid(err) && strings.Contains(err.Error(), "is forbidden") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unexpected error: %w", err)
|
||||
}).WithTimeout(15*time.Second).WithPolling(1*time.Second).Should(gomega.Succeed(), "failed waiting for DRA validation to reject the update")
|
||||
|
||||
ginkgo.By("Setting the device status a first time (via ServiceAccount with RBAC with specific driver name) for driver " + b.DriverName())
|
||||
newRole := role.DeepCopy()
|
||||
newRole.Rules = append(newRole.Rules, rbacv1.PolicyRule{
|
||||
APIGroups: []string{resourceapi.SchemeGroupVersion.Group},
|
||||
Resources: []string{"resourceclaims/driver"},
|
||||
Verbs: []string{"arbitrary-node:update", "arbitrary-node:patch"},
|
||||
ResourceNames: []string{b.DriverName()}, // allow for the specific drivers
|
||||
})
|
||||
_, err = f.ClientSet.RbacV1().ClusterRoles().Update(ctx, newRole, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Use Eventually to wait for RBAC propagation
|
||||
var updatedResourceClaim *resourceapi.ResourceClaim
|
||||
gomega.Eventually(ctx, func(ctx context.Context) error {
|
||||
var updateErr error
|
||||
updatedResourceClaim, updateErr = saClient.ResourceV1().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, allocatedResourceClaim, metav1.UpdateOptions{})
|
||||
// If it's a forbidden error, the RBAC cache hasn't updated yet, so we return the error to trigger a retry
|
||||
return updateErr
|
||||
}).WithTimeout(15*time.Second).WithPolling(1*time.Second).Should(gomega.Succeed(), "failed to update ResourceClaim status after adding RBAC rule")
|
||||
gomega.Expect(updatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim.Status.Devices).To(gomega.Equal(allocatedResourceClaim.Status.Devices))
|
||||
|
||||
ginkgo.By("Updating the device status")
|
||||
ginkgo.By("Updating the device status (via ServiceAccount with RBAC allowing all drivers)")
|
||||
newRole = role.DeepCopy()
|
||||
newRole.Rules = append(newRole.Rules, rbacv1.PolicyRule{
|
||||
APIGroups: []string{resourceapi.SchemeGroupVersion.Group},
|
||||
Resources: []string{"resourceclaims/driver"},
|
||||
Verbs: []string{"arbitrary-node:update", "arbitrary-node:patch"},
|
||||
})
|
||||
_, err = f.ClientSet.RbacV1().ClusterRoles().Update(ctx, newRole, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
updatedResourceClaim.Status.Devices[0] = resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
ShareID: shareID,
|
||||
Conditions: []metav1.Condition{{Type: "e", Status: "True", Message: "g", Reason: "h", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: &runtime.RawExtension{Raw: []byte(`{"bar":"foo"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
@@ -3174,11 +3322,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
||||
},
|
||||
}
|
||||
|
||||
updatedResourceClaim2, err := plugin.UpdateStatus(ctx, updatedResourceClaim)
|
||||
framework.ExpectNoError(err)
|
||||
var updatedResourceClaim2 *resourceapi.ResourceClaim
|
||||
gomega.Eventually(ctx, func(ctx context.Context) error {
|
||||
var updateErr error
|
||||
updatedResourceClaim2, updateErr = saClient.ResourceV1().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, updatedResourceClaim, metav1.UpdateOptions{})
|
||||
return updateErr
|
||||
}).WithTimeout(15*time.Second).WithPolling(1*time.Second).Should(gomega.Succeed(), "failed to update ResourceClaim status after wildcard RBAC rule")
|
||||
gomega.Expect(updatedResourceClaim2).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim2.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
||||
|
||||
ginkgo.By("Verifying the final status with admin client")
|
||||
getResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(getResourceClaim).ToNot(gomega.BeNil())
|
||||
@@ -3190,6 +3343,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
||||
nodes := drautils.NewNodes(f, 1, 4)
|
||||
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(1))
|
||||
driver.WithKubelet = false
|
||||
b := drautils.NewBuilder(f, driver)
|
||||
|
||||
f.It("must apply per-node permission checks", func(ctx context.Context) {
|
||||
tCtx := f.TContext(ctx)
|
||||
@@ -3334,6 +3488,79 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
||||
mustFailToDelete(fictionalNodeClient, "fictional plugin", createdClusterSlice, matchVAPDeniedError(fictionalNodeName, createdClusterSlice))
|
||||
mustDelete(f.ClientSet, "admin", createdClusterSlice)
|
||||
})
|
||||
|
||||
f.It("must be possible for a node ServiceAccount to update the node ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) {
|
||||
tCtx := f.TContext(ctx)
|
||||
claim := b.ExternalClaim()
|
||||
pod := b.PodExternal(claim.Name)
|
||||
b.Create(tCtx, claim, pod)
|
||||
|
||||
// Waits for the ResourceClaim to be allocated and the pod to be scheduled.
|
||||
b.TestPod(tCtx, pod)
|
||||
|
||||
allocatedResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(allocatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation.Devices.Results).To(gomega.HaveLen(1))
|
||||
|
||||
scheduledPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(scheduledPod).ToNot(gomega.BeNil())
|
||||
|
||||
shareID := (*string)(allocatedResourceClaim.Status.Allocation.Devices.Results[0].ShareID)
|
||||
|
||||
ginkgo.By("Setting the device status a first time")
|
||||
allocatedResourceClaim.Status.Devices = append(allocatedResourceClaim.Status.Devices,
|
||||
resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
ShareID: shareID,
|
||||
Conditions: []metav1.Condition{{Type: "a", Status: "True", Message: "c", Reason: "d", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: &runtime.RawExtension{Raw: []byte(`{"foo":"bar"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
InterfaceName: "inf1",
|
||||
IPs: []string{"10.9.8.0/24", "2001:db8::/64"},
|
||||
HardwareAddress: "bc:1c:b6:3e:b8:25",
|
||||
},
|
||||
})
|
||||
|
||||
// Updates the ResourceClaim from the driver on the same node as the pod.
|
||||
plugin, ok := driver.Nodes[scheduledPod.Spec.NodeName]
|
||||
if !ok {
|
||||
framework.Failf("pod got scheduled to node %s without a plugin", scheduledPod.Spec.NodeName)
|
||||
}
|
||||
updatedResourceClaim, err := plugin.UpdateStatus(ctx, allocatedResourceClaim)
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(updatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim.Status.Devices).To(gomega.Equal(allocatedResourceClaim.Status.Devices))
|
||||
|
||||
ginkgo.By("Updating the device status")
|
||||
updatedResourceClaim.Status.Devices[0] = resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
ShareID: shareID,
|
||||
Conditions: []metav1.Condition{{Type: "e", Status: "True", Message: "g", Reason: "h", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: &runtime.RawExtension{Raw: []byte(`{"bar":"foo"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
InterfaceName: "inf2",
|
||||
IPs: []string{"10.9.8.1/24", "2001:db8::1/64"},
|
||||
HardwareAddress: "bc:1c:b6:3e:b8:26",
|
||||
},
|
||||
}
|
||||
|
||||
updatedResourceClaim2, err := plugin.UpdateStatus(ctx, updatedResourceClaim)
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(updatedResourceClaim2).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim2.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
||||
|
||||
getResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(getResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(getResourceClaim.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
||||
})
|
||||
})
|
||||
|
||||
multipleDrivers := func(nodeV1beta1, nodeV1 bool) {
|
||||
|
||||
@@ -17,7 +17,11 @@ rules:
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims/status"]
|
||||
verbs: ["update"]
|
||||
verbs: ["patch", "update"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims/driver"]
|
||||
verbs: ["associated-node:patch", "associated-node:update"]
|
||||
resourceNames: ["dra-kubelet-plugin-driver-name"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get"]
|
||||
|
||||
@@ -82,6 +82,11 @@ func (b *Builder) ClassName() string {
|
||||
return b.namespace + b.Driver.NameSuffix + "-class"
|
||||
}
|
||||
|
||||
// DriverName returns the default device driver name.
|
||||
func (b *Builder) DriverName() string {
|
||||
return b.Driver.Name
|
||||
}
|
||||
|
||||
// Class returns the device Class that the builder's other objects
|
||||
// reference.
|
||||
func (b *Builder) Class() *DeviceClassWrapper {
|
||||
|
||||
@@ -474,7 +474,9 @@ func (d *Driver) SetUp(tCtx ktesting.TContext, kubeletRootDir string, nodes *Nod
|
||||
// Create service account and corresponding RBAC rules.
|
||||
d.serviceAccountName = "dra-kubelet-plugin-" + d.Name + d.InstanceSuffix + "-service-account"
|
||||
content := example.PluginPermissions
|
||||
|
||||
content = strings.ReplaceAll(content, "dra-kubelet-plugin-namespace", tCtx.Namespace())
|
||||
content = strings.ReplaceAll(content, "dra-kubelet-plugin-driver-name", d.Name)
|
||||
content = strings.ReplaceAll(content, "dra-kubelet-plugin", "dra-kubelet-plugin-"+d.Name+d.InstanceSuffix)
|
||||
d.createFromYAML(tCtx, []byte(content), tCtx.Namespace())
|
||||
|
||||
|
||||
295
test/integration/auth/resourceclaim_test.go
Normal file
295
test/integration/auth/resourceclaim_test.go
Normal file
@@ -0,0 +1,295 @@
|
||||
/*
|
||||
Copyright The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
resourceapi "k8s.io/api/resource/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/test/integration/framework"
|
||||
)
|
||||
|
||||
func TestResourceClaimGranularStatusAuthorization(t *testing.T) {
|
||||
// Enable Feature Gates Globally for the test run
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicResourceAllocation, true)
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAResourceClaimDeviceStatus, true)
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAResourceClaimGranularStatusAuthorization, true)
|
||||
|
||||
const (
|
||||
ns = "dra-authz-test"
|
||||
saName = "dra-plugin-sa"
|
||||
claimName = "test-claim"
|
||||
nodeName = "worker-1"
|
||||
)
|
||||
|
||||
testcases := []struct {
|
||||
name string
|
||||
preAllocate bool
|
||||
impersonateExtra map[string][]string
|
||||
setupRBAC func(t *testing.T, adminClient clientset.Interface)
|
||||
updateClaim func(c *resourceapi.ResourceClaim)
|
||||
verifyErr func(t *testing.T, err error)
|
||||
}{
|
||||
{
|
||||
name: "fails to update status.devices without driver permission",
|
||||
preAllocate: true,
|
||||
setupRBAC: func(t *testing.T, adminClient clientset.Interface) {}, // No extra RBAC beyond front-door
|
||||
updateClaim: func(c *resourceapi.ResourceClaim) {
|
||||
c.Status.Devices = []resourceapi.AllocatedDeviceStatus{
|
||||
{Driver: "test-driver", Pool: "pool1", Device: "dev1"},
|
||||
}
|
||||
},
|
||||
verifyErr: func(t *testing.T, err error) {
|
||||
if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "Forbidden: changing status.devices requires") {
|
||||
t.Errorf("Expected Invalid/Forbidden error, got: %v", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "succeeds with associated-node permission for same-node SA",
|
||||
preAllocate: true,
|
||||
impersonateExtra: map[string][]string{
|
||||
"authentication.kubernetes.io/node-name": {nodeName},
|
||||
},
|
||||
setupRBAC: func(t *testing.T, adminClient clientset.Interface) {
|
||||
createRoleAndBinding(t, adminClient, ns, saName, "node-local-driver",
|
||||
[]string{"resourceclaims/driver"}, []string{"associated-node:update"})
|
||||
},
|
||||
updateClaim: func(c *resourceapi.ResourceClaim) {
|
||||
c.Status.Devices = []resourceapi.AllocatedDeviceStatus{
|
||||
{Driver: "test-driver", Pool: "pool1", Device: "dev1"},
|
||||
}
|
||||
},
|
||||
verifyErr: func(t *testing.T, err error) {
|
||||
if err != nil {
|
||||
t.Errorf("Expected success via associated-node, got: %v", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fails deallocation without binding permission",
|
||||
preAllocate: true,
|
||||
setupRBAC: func(t *testing.T, adminClient clientset.Interface) {},
|
||||
updateClaim: func(c *resourceapi.ResourceClaim) {
|
||||
c.Status.Allocation = nil
|
||||
},
|
||||
verifyErr: func(t *testing.T, err error) {
|
||||
if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "Forbidden: changing status.allocation") {
|
||||
t.Errorf("Expected Invalid/Forbidden on unbind, got: %v", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "succeeds to update status.reservedFor with binding permission",
|
||||
preAllocate: true,
|
||||
setupRBAC: func(t *testing.T, adminClient clientset.Interface) {
|
||||
createClusterRoleAndBinding(t, adminClient, ns, saName, "cluster-binding-updater-reserved",
|
||||
[]string{"resourceclaims/binding"}, []string{"update"})
|
||||
},
|
||||
updateClaim: func(c *resourceapi.ResourceClaim) {
|
||||
c.Status.ReservedFor = []resourceapi.ResourceClaimConsumerReference{
|
||||
{Resource: "pods", Name: "pod-1", UID: "uid-1"},
|
||||
}
|
||||
},
|
||||
verifyErr: func(t *testing.T, err error) {
|
||||
if err != nil {
|
||||
t.Errorf("Expected success, got: %v", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fails when updating both allocation and devices but missing binding permission",
|
||||
preAllocate: true,
|
||||
setupRBAC: func(t *testing.T, adminClient clientset.Interface) {
|
||||
// Has driver permission, but LACKS binding permission
|
||||
createRoleAndBinding(t, adminClient, ns, saName, "driver-only",
|
||||
[]string{"resourceclaims/driver"}, []string{"arbitrary-node:update"})
|
||||
},
|
||||
updateClaim: func(c *resourceapi.ResourceClaim) {
|
||||
// Re-allocate to a different node (requires binding)
|
||||
if c.Status.Allocation != nil && c.Status.Allocation.NodeSelector != nil {
|
||||
c.Status.Allocation.NodeSelector.NodeSelectorTerms[0].MatchFields[0].Values = []string{"worker-2"}
|
||||
}
|
||||
// Change devices (requires driver)
|
||||
c.Status.Devices = []resourceapi.AllocatedDeviceStatus{
|
||||
{Driver: "test-driver", Pool: "pool1", Device: "dev2"},
|
||||
}
|
||||
},
|
||||
verifyErr: func(t *testing.T, err error) {
|
||||
if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "Forbidden: changing status.allocation") {
|
||||
t.Errorf("Expected Forbidden on simultaneous update missing binding, got: %v", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{
|
||||
"--runtime-config=api/all=true",
|
||||
"--authorization-mode=RBAC",
|
||||
}, framework.SharedEtcd())
|
||||
t.Cleanup(server.TearDownFn)
|
||||
|
||||
adminClient := clientset.NewForConfigOrDie(server.ClientConfig)
|
||||
|
||||
// Setup Namespace and Service Account
|
||||
_, err := adminClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = adminClient.CoreV1().ServiceAccounts(ns).Create(context.TODO(), &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName}}, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create the base ResourceClaim
|
||||
claim := &resourceapi.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: claimName},
|
||||
Spec: resourceapi.ResourceClaimSpec{
|
||||
Devices: resourceapi.DeviceClaim{
|
||||
Requests: []resourceapi.DeviceRequest{{
|
||||
Name: "req-1",
|
||||
FirstAvailable: []resourceapi.DeviceSubRequest{{
|
||||
Name: "subreq-1",
|
||||
DeviceClassName: "test-class",
|
||||
}},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = adminClient.ResourceV1().ResourceClaims(ns).Create(context.TODO(), claim, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Admin Pre-allocation (if required by test)
|
||||
if tc.preAllocate {
|
||||
c, err := adminClient.ResourceV1().ResourceClaims(ns).Get(context.TODO(), claimName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to fetch claim for pre-allocation: %v", err)
|
||||
}
|
||||
c.Status.Allocation = &resourceapi.AllocationResult{
|
||||
NodeSelector: &corev1.NodeSelector{
|
||||
NodeSelectorTerms: []corev1.NodeSelectorTerm{{
|
||||
MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{nodeName}}},
|
||||
}},
|
||||
},
|
||||
Devices: resourceapi.DeviceAllocationResult{
|
||||
Results: []resourceapi.DeviceRequestAllocationResult{
|
||||
{Request: "req-1", Driver: "test-driver", Pool: "pool1", Device: "dev1"},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = adminClient.ResourceV1().ResourceClaims(ns).UpdateStatus(context.TODO(), c, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Admin failed to set baseline allocation: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Setup RBAC
|
||||
createRoleAndBinding(t, adminClient, ns, saName, "base-status-updater", []string{"resourceclaims/status"}, []string{"update", "patch"})
|
||||
createRoleAndBinding(t, adminClient, ns, saName, "base-claim-reader", []string{"resourceclaims"}, []string{"get"})
|
||||
tc.setupRBAC(t, adminClient)
|
||||
|
||||
// Build the Impersonated Client
|
||||
saConfig := rest.CopyConfig(server.ClientConfig)
|
||||
saConfig.Impersonate = rest.ImpersonationConfig{
|
||||
UserName: fmt.Sprintf("system:serviceaccount:%s:%s", ns, saName),
|
||||
Extra: tc.impersonateExtra,
|
||||
}
|
||||
saClient := clientset.NewForConfigOrDie(saConfig)
|
||||
|
||||
// Execute Test Update
|
||||
cToUpdate, err := adminClient.ResourceV1().ResourceClaims(ns).Get(context.TODO(), claimName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to fetch claim before test execution: %v", err)
|
||||
}
|
||||
tc.updateClaim(cToUpdate)
|
||||
_, testErr := saClient.ResourceV1().ResourceClaims(ns).UpdateStatus(context.TODO(), cToUpdate, metav1.UpdateOptions{})
|
||||
|
||||
// 7. Verify Results
|
||||
tc.verifyErr(t, testErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// createRoleAndBinding is a quick helper to assign namespaced RBAC rules
|
||||
func createRoleAndBinding(t *testing.T, client clientset.Interface, ns, saName, roleName string, resources, verbs []string) {
|
||||
role := &rbacv1.Role{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: roleName},
|
||||
Rules: []rbacv1.PolicyRule{{
|
||||
APIGroups: []string{"resource.k8s.io"},
|
||||
Resources: resources,
|
||||
Verbs: verbs,
|
||||
}},
|
||||
}
|
||||
_, err := client.RbacV1().Roles(ns).Create(context.TODO(), role, metav1.CreateOptions{})
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
binding := &rbacv1.RoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: roleName + "-binding"},
|
||||
Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: ns}},
|
||||
RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: roleName},
|
||||
}
|
||||
_, err = client.RbacV1().RoleBindings(ns).Create(context.TODO(), binding, metav1.CreateOptions{})
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// createClusterRoleAndBinding is a helper for cluster-scoped synthetic checks (like binding)
|
||||
func createClusterRoleAndBinding(t *testing.T, client clientset.Interface, ns, saName, roleName string, resources, verbs []string) {
|
||||
role := &rbacv1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: roleName},
|
||||
Rules: []rbacv1.PolicyRule{{
|
||||
APIGroups: []string{"resource.k8s.io"},
|
||||
Resources: resources,
|
||||
Verbs: verbs,
|
||||
}},
|
||||
}
|
||||
_, err := client.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{})
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
binding := &rbacv1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: roleName + "-binding"},
|
||||
Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: ns}},
|
||||
RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: roleName},
|
||||
}
|
||||
_, err = client.RbacV1().ClusterRoleBindings().Create(context.TODO(), binding, metav1.CreateOptions{})
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user