diff --git a/test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml b/test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml index 8960f3b0080..0877b16e93b 100644 --- a/test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml +++ b/test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml @@ -31,6 +31,9 @@ spec: - name: dev hostPath: path: /dev + - name: cdi-dir + hostPath: + path: /var/run/cdi containers: - image: registry.k8s.io/e2e-test-images/sample-device-plugin:1.3 name: sample-device-plugin @@ -46,5 +49,7 @@ spec: mountPath: /var/lib/kubelet/plugins_registry - name: dev mountPath: /dev + - name: cdi-dir + mountPath: /var/run/cdi updateStrategy: type: RollingUpdate diff --git a/test/e2e_node/device_plugin_test.go b/test/e2e_node/device_plugin_test.go index 98db64fc40a..e16e0681af5 100644 --- a/test/e2e_node/device_plugin_test.go +++ b/test/e2e_node/device_plugin_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" admissionapi "k8s.io/pod-security-admission/api" @@ -47,9 +48,11 @@ import ( "k8s.io/kubectl/pkg/util/podutils" kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" kubeletpodresourcesv1alpha1 "k8s.io/kubelet/pkg/apis/podresources/v1alpha1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" ) @@ -239,6 +242,16 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices[0].DeviceIds).To(gomega.HaveLen(1)) }) + ginkgo.It("can make a CDI device accessible in a container", func(ctx context.Context) { + e2eskipper.SkipUnlessFeatureGateEnabled(features.DevicePluginCDIDevices) + // check if CDI_DEVICE env variable is set + // and only one correspondent device node /tmp/ is available inside a container + podObj := makeBusyboxPod(SampleDeviceResourceName, "[ $(ls /tmp/CDI-Dev-[1,2] | wc -l) -eq 1 -a -b /tmp/$CDI_DEVICE ]") + podObj.Spec.RestartPolicy = v1.RestartPolicyNever + pod := e2epod.NewPodClient(f).Create(ctx, podObj) + framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)) + }) + // simulate container restart, while all other involved components (kubelet, device plugin) stay stable. To do so, in the container // entry point we sleep for a limited and short period of time. The device assignment should be kept and be stable across the container // restarts. For the sake of brevity we however check just the fist restart. @@ -906,6 +919,10 @@ func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod { } } + if utilfeature.DefaultFeatureGate.Enabled(features.DevicePluginCDIDevices) { + dp.Spec.Containers[0].Env = append(dp.Spec.Containers[0].Env, v1.EnvVar{Name: "CDI_ENABLED", Value: "1"}) + } + return dp } diff --git a/test/images/sample-device-plugin/sampledeviceplugin b/test/images/sample-device-plugin/sampledeviceplugin new file mode 100755 index 00000000000..86b045b90ae Binary files /dev/null and b/test/images/sample-device-plugin/sampledeviceplugin differ diff --git a/test/images/sample-device-plugin/sampledeviceplugin.go b/test/images/sample-device-plugin/sampledeviceplugin.go index 1f78c890a32..ccdb0c7a9ab 100644 --- a/test/images/sample-device-plugin/sampledeviceplugin.go +++ b/test/images/sample-device-plugin/sampledeviceplugin.go @@ -32,6 +32,9 @@ import ( const ( resourceName = "example.com/resource" + cdiPath = "/var/run/cdi/example.com.json" + cdiVersion = "0.3.0" + cdiPrefix = "CDI-" ) // stubAllocFunc creates and returns allocation response for the input allocate request @@ -68,6 +71,14 @@ func stubAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Devic ContainerPath: fpath, HostPath: fpath, }) + + if os.Getenv("CDI_ENABLED") != "" { + // add the CDI device ID to the response. + cdiDevice := &pluginapi.CDIDevice{ + Name: fmt.Sprintf("%s=%s", resourceName, cdiPrefix+dev.ID), + } + response.CDIDevices = append(response.CDIDevices, cdiDevice) + } } responses.ContainerResponses = append(responses.ContainerResponses, response) } @@ -104,6 +115,24 @@ func main() { } dp1.SetAllocFunc(stubAllocFunc) + + cdiEnabled := os.Getenv("CDI_ENABLED") + klog.Infof("CDI_ENABLED: %s", cdiEnabled) + if cdiEnabled != "" { + if err := createCDIFile(devs); err != nil { + panic(err) + } + defer func() { + // Remove CDI file + if _, err := os.Stat(cdiPath); err == nil || os.IsExist(err) { + err := os.Remove(cdiPath) + if err != nil { + panic(err) + } + } + }() + } + var registerControlFile string autoregister := true @@ -200,3 +229,20 @@ func handleRegistrationProcess(registerControlFile string, dpStub *plugin.Stub, } } } + +func createCDIFile(devs []*pluginapi.Device) error { + content := fmt.Sprintf(`{"cdiVersion":"%s","kind":"%s","devices":[`, cdiVersion, resourceName) + for i, dev := range devs { + name := cdiPrefix + dev.ID + content += fmt.Sprintf(`{"name":"%s","containerEdits":{"env":["CDI_DEVICE=%s"],"deviceNodes":[{"path":"/tmp/%s","type":"b","major":1,"minor":%d}]}}`, name, name, name, i) + if i < len(devs)-1 { + content += "," + } + } + content += "]}" + if err := os.WriteFile(cdiPath, []byte(content), 0644); err != nil { + return fmt.Errorf("failed to create CDI file: %s", err) + } + klog.InfoS("Created CDI file", "path", cdiPath, "devices", devs) + return nil +}