Merge pull request #121440 from bart0sh/PR125-e2e-test-DevicePluginCDIDevices

e2e_node: test device plugin support of CDI devices

test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml

@@ -31,6 +31,9 @@ spec:
       - name: dev
         hostPath:
           path: /dev
+      - name: cdi-dir
+        hostPath:
+          path: /var/run/cdi
       containers:
       - image: registry.k8s.io/e2e-test-images/sample-device-plugin:1.3
         name: sample-device-plugin
@@ -46,5 +49,7 @@ spec:
           mountPath: /var/lib/kubelet/plugins_registry
         - name: dev
           mountPath: /dev
+        - name: cdi-dir
+          mountPath: /var/run/cdi
   updateStrategy:
     type: RollingUpdate

test/e2e_node/device_plugin_test.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 	admissionapi "k8s.io/pod-security-admission/api"
@@ -47,9 +48,11 @@ import (
 	"k8s.io/kubectl/pkg/util/podutils"
 	kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
 	kubeletpodresourcesv1alpha1 "k8s.io/kubelet/pkg/apis/podresources/v1alpha1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
@@ -239,6 +242,16 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 			gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices[0].DeviceIds).To(gomega.HaveLen(1))
 		})
 
+		ginkgo.It("can make a CDI device accessible in a container", func(ctx context.Context) {
+			e2eskipper.SkipUnlessFeatureGateEnabled(features.DevicePluginCDIDevices)
+			// check if CDI_DEVICE env variable is set
+			// and only one correspondent device node /tmp/<CDI_DEVICE> is available inside a container
+			podObj := makeBusyboxPod(SampleDeviceResourceName, "[ $(ls /tmp/CDI-Dev-[1,2] | wc -l) -eq 1 -a -b /tmp/$CDI_DEVICE ]")
+			podObj.Spec.RestartPolicy = v1.RestartPolicyNever
+			pod := e2epod.NewPodClient(f).Create(ctx, podObj)
+			framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
+		})
+
 		// simulate container restart, while all other involved components (kubelet, device plugin) stay stable. To do so, in the container
 		// entry point we sleep for a limited and short period of time. The device assignment should be kept and be stable across the container
 		// restarts. For the sake of brevity we however check just the fist restart.
@@ -906,6 +919,10 @@ func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod {
 		}
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.DevicePluginCDIDevices) {
+		dp.Spec.Containers[0].Env = append(dp.Spec.Containers[0].Env, v1.EnvVar{Name: "CDI_ENABLED", Value: "1"})
+	}
+
 	return dp
 }
 

test/images/sample-device-plugin/sampledeviceplugin.go

@@ -32,6 +32,9 @@ import (
 
 const (
 	resourceName = "example.com/resource"
+	cdiPath      = "/var/run/cdi/example.com.json"
+	cdiVersion   = "0.3.0"
+	cdiPrefix    = "CDI-"
 )
 
 // stubAllocFunc creates and returns allocation response for the input allocate request
@@ -68,6 +71,14 @@ func stubAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Devic
 				ContainerPath: fpath,
 				HostPath:      fpath,
 			})
+
+			if os.Getenv("CDI_ENABLED") != "" {
+				// add the CDI device ID to the response.
+				cdiDevice := &pluginapi.CDIDevice{
+					Name: fmt.Sprintf("%s=%s", resourceName, cdiPrefix+dev.ID),
+				}
+				response.CDIDevices = append(response.CDIDevices, cdiDevice)
+			}
 		}
 		responses.ContainerResponses = append(responses.ContainerResponses, response)
 	}
@@ -104,6 +115,24 @@ func main() {
 
 	}
 	dp1.SetAllocFunc(stubAllocFunc)
+
+	cdiEnabled := os.Getenv("CDI_ENABLED")
+	klog.Infof("CDI_ENABLED: %s", cdiEnabled)
+	if cdiEnabled != "" {
+		if err := createCDIFile(devs); err != nil {
+			panic(err)
+		}
+		defer func() {
+			// Remove CDI file
+			if _, err := os.Stat(cdiPath); err == nil || os.IsExist(err) {
+				err := os.Remove(cdiPath)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}()
+	}
+
 	var registerControlFile string
 	autoregister := true
 
@@ -200,3 +229,20 @@ func handleRegistrationProcess(registerControlFile string, dpStub *plugin.Stub,
 		}
 	}
 }
+
+func createCDIFile(devs []*pluginapi.Device) error {
+	content := fmt.Sprintf(`{"cdiVersion":"%s","kind":"%s","devices":[`, cdiVersion, resourceName)
+	for i, dev := range devs {
+		name := cdiPrefix + dev.ID
+		content += fmt.Sprintf(`{"name":"%s","containerEdits":{"env":["CDI_DEVICE=%s"],"deviceNodes":[{"path":"/tmp/%s","type":"b","major":1,"minor":%d}]}}`, name, name, name, i)
+		if i < len(devs)-1 {
+			content += ","
+		}
+	}
+	content += "]}"
+	if err := os.WriteFile(cdiPath, []byte(content), 0644); err != nil {
+		return fmt.Errorf("failed to create CDI file: %s", err)
+	}
+	klog.InfoS("Created CDI file", "path", cdiPath, "devices", devs)
+	return nil
+}