Add CNIDeviceFile in RuntimeConfig

DPDeviceFile is used by Device Plugins to pass device data to CNIs, as defined
in the device-info-spec. The name of the DPDeviceFile is defined by the
device-info-spec as:
 <ResourceName>-<DeviceID>-device.json

If the DPDeviceFile exists, the NPWG implementation makes a copy of the file
and passes the name of the file to the delegate CNI via capabilityArgs as
CNIDeviceFile. If the DPDeviceFile does not exist, the filename is still
passed to the CNI. The CNI can create the file and popluate it if a device
is created within the CNI.

The name of the CNIDeviceFile is not defined by device-info-spec, but to
ensure the name does not clash it is formed by the following unique triplet:
[networkName, PodUUID, ifName]

k8snetworkplumbingwg/network-attachment-definition-client repo has utility
functions to abstract some of this functionality so it can be reused across
Device Plugins, NPWG implementations and CNIs.

Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com>

k8sclient/k8sclient.go

@@ -285,7 +285,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		return nil, resourceMap, err
 	}
 
-	delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID)
+	delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID, resourceName)
 	if err != nil {
 		return nil, resourceMap, err
 	}
@@ -495,7 +495,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 	var configBytes []byte
 	configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
 	if err == nil {
-		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
 		if err != nil {
 			return nil, resourceMap, err
 		}
@@ -514,7 +514,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 				var configBytes []byte
 				configBytes, err = netutils.GetCNIConfigFromFile("", netname)
 				if err == nil {
-					delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+					delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
 					if err != nil {
 						return nil, resourceMap, err
 					}

k8sclient/k8sclient_test.go

@@ -1184,7 +1184,7 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
 			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1266,7 +1266,7 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
 			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1326,7 +1326,7 @@ users:
 		}`
 			// note that the provided kubeconfig is invalid
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
 			Expect(err).NotTo(HaveOccurred())
 
 			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1385,7 +1385,7 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
 			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)

multus/multus.go

@@ -455,11 +455,19 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
 		ifName := getIfname(delegates[idx], args.IfName, idx)
-		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
+		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
 		if err := delegateDel(exec, pod, ifName, delegates[idx], rt, binDir); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
+		if cniDeviceInfoPath != "" {
+			err := nadutils.CleanDeviceInfoForCNI(cniDeviceInfoPath)
+			// Even if the filename is set, file may not be present. Ignore error,
+			// but log and in the future may need to filter on specific errors.
+			if err != nil {
+				logging.Debugf("delPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err)
+			}
+		}
 	}
 
 	// Check if we had any errors, and send them all back.
@@ -565,8 +573,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
+		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
+		if cniDeviceInfoPath != "" {
+			err = nadutils.CopyDeviceInfoForCNIFromDP(cniDeviceInfoPath, delegate.ResourceName, delegate.DeviceID)
+			// Even if the filename is set, file may not be present. Ignore error,
+			// but log and in the future may need to filter on specific errors.
+			if err != nil {
+				logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
+			}
+		}
 
-		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
 		tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n.BinDir, cniArgs)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
@@ -658,7 +674,7 @@ func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 	for idx, delegate := range in.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
 
-		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
 		err = delegateCheck(exec, ifName, delegate, rt, in.BinDir)
 		if err != nil {
 			return err

multus/multus_test.go

@@ -2031,7 +2031,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
 		k8sargs, err := k8sclient.GetK8sArgs(args)
 		n, err := types.LoadNetConf(args.StdinData)
-		rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
 
 		err = conflistDel(rt, rawnetconflist, binDir, fExec)
 		Expect(err).To(HaveOccurred())
@@ -3277,7 +3277,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`)
 		k8sargs, err := k8sclient.GetK8sArgs(args)
 		n, err := types.LoadNetConf(args.StdinData)
-		rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
 
 		err = conflistDel(rt, rawnetconflist, binDir, fExec)
 		Expect(err).To(HaveOccurred())

types/conf.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/cni/pkg/version"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"gopkg.in/intel/multus-cni.v3/logging"
 )
 
@@ -55,7 +56,7 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 }
 
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
-func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string) (*DelegateNetConf, error) {
+func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
 	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
 
@@ -87,6 +88,9 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConf bytes: %v", err)
 			}
+			// Save them for housekeeping
+			delegateConf.ResourceName = resourceName
+			delegateConf.DeviceID = deviceID
 		}
 		if net != nil && net.CNIArgs != nil {
 			bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs)
@@ -164,6 +168,7 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
 		if delegate.DeviceID != "" {
 			runtimeConfig.DeviceID = delegate.DeviceID
 		}
+		logging.Debugf("mergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", runtimeConfig)
 	}
 
 	return runtimeConfig
@@ -171,12 +176,19 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
 
 // CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
 // exists, merge data with the runtime config.
-func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) *libcni.RuntimeConf {
+func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
 	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
+	var cniDeviceInfoFile string
 
 	delegateRc := rc
 	if delegate != nil {
 		delegateRc = mergeCNIRuntimeConfig(delegateRc, delegate)
+		if delegateRc.CNIDeviceInfoFile == "" && delegate.Name != "" {
+			autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName)
+			delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo)
+			cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile
+			logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile)
+		}
 	}
 
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
@@ -215,9 +227,12 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 		if delegateRc.DeviceID != "" {
 			capabilityArgs["deviceID"] = delegateRc.DeviceID
 		}
+		if delegateRc.CNIDeviceInfoFile != "" {
+			capabilityArgs["CNIDeviceInfoFile"] = delegateRc.CNIDeviceInfoFile
+		}
 		rt.CapabilityArgs = capabilityArgs
 	}
-	return rt
+	return rt, cniDeviceInfoFile
 }
 
 // GetGatewayFromResult retrieves gateway IP addresses from CNI result
@@ -311,7 +326,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 			if err != nil {
 				return nil, logging.Errorf("LoadNetConf: error marshalling delegate %d config: %v", idx, err)
 			}
-			delegateConf, err := LoadDelegateNetConf(bytes, nil, "")
+			delegateConf, err := LoadDelegateNetConf(bytes, nil, "", "")
 			if err != nil {
 				return nil, logging.Errorf("LoadNetConf: failed to load delegate %d config: %v", idx, err)
 			}

types/conf_test.go

@@ -102,7 +102,7 @@ var _ = Describe("config operations", func() {
 
 		_, err := LoadNetConf([]byte(conf))
 		Expect(err).To(HaveOccurred())
-		_, err = LoadDelegateNetConf([]byte(conf), nil, "")
+		_, err = LoadDelegateNetConf([]byte(conf), nil, "", "")
 		Expect(err).To(HaveOccurred())
 		err = LoadDelegateNetConfList([]byte(conf), &DelegateNetConf{})
 		Expect(err).To(HaveOccurred())
@@ -280,7 +280,7 @@ var _ = Describe("config operations", func() {
 			DeviceID string `json:"deviceID"`
 		}
 		sriovConf := &sriovNetConf{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConf)
@@ -306,7 +306,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*sriovNetConf `json:"plugins"`
 		}
 		sriovConfList := &sriovNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -335,7 +335,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*sriovNetConf `json:"plugins"`
 		}
 		sriovConfList := &sriovNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -356,7 +356,7 @@ var _ = Describe("config operations", func() {
 			PCIBusID string `json:"pciBusID"`
 		}
 		hostDeviceConf := &hostDeviceNetConf{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConf)
@@ -382,7 +382,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*hostDeviceNetConf `json:"plugins"`
 		}
 		hostDeviceConfList := &hostDeviceNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -411,7 +411,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*hostDeviceNetConf `json:"plugins"`
 		}
 		hostDeviceConfList := &hostDeviceNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -444,7 +444,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConf := &bridgeNetConf{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -480,7 +480,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConf := &bridgeNetConf{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -518,7 +518,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConflist := &bridgeNetConfList{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConflist)
@@ -565,7 +565,7 @@ var _ = Describe("config operations", func() {
 			HostIP:        "anotherSampleHostIP",
 		}
 
-		rt := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil)
+		rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil)
 		fmt.Println("rt.ContainerID: ", rt.ContainerID)
 		Expect(rt.ContainerID).To(Equal("123456789"))
 		Expect(rt.NetNS).To(Equal(args.Netns))
@@ -595,7 +595,7 @@ var _ = Describe("config operations", func() {
     }
 }`
 
-		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -628,7 +628,7 @@ var _ = Describe("config operations", func() {
     }
 }`
 
-		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 		fmt.Println("result.Version: ", result.Version())
 		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -668,7 +668,7 @@ var _ = Describe("config operations", func() {
 			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
 
-		delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "")
+		delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		Expect(delegateConf.IfnameRequest).To(Equal(networkSelection.InterfaceRequest))
 		Expect(delegateConf.MacRequest).To(Equal(networkSelection.MacRequest))
@@ -714,7 +714,7 @@ var _ = Describe("config operations", func() {
 			BandwidthRequest:      bandwidthEntry1,
 			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
-		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
+		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
 		delegate.MasterPlugin = true
 		Expect(err).NotTo(HaveOccurred())
 		runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
@@ -751,7 +751,7 @@ var _ = Describe("config operations", func() {
 			BandwidthRequest:      bandwidthEntry1,
 			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
-		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
+		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
 		Expect(runtimeConf.PortMaps).NotTo(BeNil())

types/types.go

@@ -63,6 +63,7 @@ type RuntimeConfig struct {
 	Mac               string          `json:"mac,omitempty"`
 	InfinibandGUID    string          `json:"infinibandGUID,omitempty"`
 	DeviceID          string          `json:"deviceID,omitempty"`
+	CNIDeviceInfoFile string          `json:"CNIDeviceInfoFile,omitempty"`
 }
 
 // PortMapEntry for CNI PortMapEntry
@@ -111,6 +112,8 @@ type DelegateNetConf struct {
 	ConfListPlugin bool `json:"-"`
 	// DeviceID is only used internal housekeeping
 	DeviceID string `json:"deviceID,omitempty"`
+	// ResourceName is only used internal housekeeping
+	ResourceName string `json:"resourceName,omitempty"`
 
 	// Raw JSON
 	Bytes []byte