diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 858085c28..e0b96067e 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -285,7 +285,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen return nil, resourceMap, err } - delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID) + delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID, resourceName) if err != nil { return nil, resourceMap, err } @@ -495,7 +495,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace var configBytes []byte configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir) if err == nil { - delegate, err := types.LoadDelegateNetConf(configBytes, nil, "") + delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "") if err != nil { return nil, resourceMap, err } @@ -514,7 +514,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace var configBytes []byte configBytes, err = netutils.GetCNIConfigFromFile("", netname) if err == nil { - delegate, err := types.LoadDelegateNetConf(configBytes, nil, "") + delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "") if err != nil { return nil, resourceMap, err } diff --git a/k8sclient/k8sclient_test.go b/k8sclient/k8sclient_test.go index 05ae40675..8f8ba5550 100644 --- a/k8sclient/k8sclient_test.go +++ b/k8sclient/k8sclient_test.go @@ -1184,7 +1184,7 @@ users: } }` - delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) @@ -1266,7 +1266,7 @@ users: } }` - delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) @@ -1326,7 +1326,7 @@ users: }` // note that the provided kubeconfig is invalid - delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "") + delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "") Expect(err).NotTo(HaveOccurred()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) @@ -1385,7 +1385,7 @@ users: } }` - delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) diff --git a/multus/multus.go b/multus/multus.go index 2a5326a07..b41b7aef8 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -455,11 +455,19 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type var errorstrings []string for idx := lastIdx; idx >= 0; idx-- { ifName := getIfname(delegates[idx], args.IfName, idx) - rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx]) + rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx]) // Attempt to delete all but do not error out, instead, collect all errors. if err := delegateDel(exec, pod, ifName, delegates[idx], rt, binDir); err != nil { errorstrings = append(errorstrings, err.Error()) } + if cniDeviceInfoPath != "" { + err := nadutils.CleanDeviceInfoForCNI(cniDeviceInfoPath) + // Even if the filename is set, file may not be present. Ignore error, + // but log and in the future may need to filter on specific errors. + if err != nil { + logging.Debugf("delPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err) + } + } } // Check if we had any errors, and send them all back. @@ -565,8 +573,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c cniArgs := os.Getenv("CNI_ARGS") for idx, delegate := range n.Delegates { ifName := getIfname(delegate, args.IfName, idx) + rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate) + if cniDeviceInfoPath != "" { + err = nadutils.CopyDeviceInfoForCNIFromDP(cniDeviceInfoPath, delegate.ResourceName, delegate.DeviceID) + // Even if the filename is set, file may not be present. Ignore error, + // but log and in the future may need to filter on specific errors. + if err != nil { + logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err) + } + } - rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate) tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n.BinDir, cniArgs) if err != nil { // If the add failed, tear down all networks we already added @@ -658,7 +674,7 @@ func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) for idx, delegate := range in.Delegates { ifName := getIfname(delegate, args.IfName, idx) - rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate) + rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate) err = delegateCheck(exec, ifName, delegate, rt, in.BinDir) if err != nil { return err diff --git a/multus/multus_test.go b/multus/multus_test.go index 773b929d0..0184e597b 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -2031,7 +2031,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() { rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`) k8sargs, err := k8sclient.GetK8sArgs(args) n, err := types.LoadNetConf(args.StdinData) - rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil) + rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil) err = conflistDel(rt, rawnetconflist, binDir, fExec) Expect(err).To(HaveOccurred()) @@ -3277,7 +3277,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() { rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`) k8sargs, err := k8sclient.GetK8sArgs(args) n, err := types.LoadNetConf(args.StdinData) - rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil) + rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil) err = conflistDel(rt, rawnetconflist, binDir, fExec) Expect(err).To(HaveOccurred()) diff --git a/types/conf.go b/types/conf.go index 495a47836..16e480707 100644 --- a/types/conf.go +++ b/types/conf.go @@ -24,6 +24,7 @@ import ( "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" + nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils" "gopkg.in/intel/multus-cni.v3/logging" ) @@ -55,7 +56,7 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error } // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure -func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string) (*DelegateNetConf, error) { +func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) { var err error logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID) @@ -87,6 +88,9 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st if err != nil { return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConf bytes: %v", err) } + // Save them for housekeeping + delegateConf.ResourceName = resourceName + delegateConf.DeviceID = deviceID } if net != nil && net.CNIArgs != nil { bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs) @@ -164,6 +168,7 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo if delegate.DeviceID != "" { runtimeConfig.DeviceID = delegate.DeviceID } + logging.Debugf("mergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", runtimeConfig) } return runtimeConfig @@ -171,12 +176,19 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo // CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration // exists, merge data with the runtime config. -func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) *libcni.RuntimeConf { +func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) { logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate) + var cniDeviceInfoFile string delegateRc := rc if delegate != nil { delegateRc = mergeCNIRuntimeConfig(delegateRc, delegate) + if delegateRc.CNIDeviceInfoFile == "" && delegate.Name != "" { + autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName) + delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo) + cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile + logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile) + } } // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf @@ -215,9 +227,12 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r if delegateRc.DeviceID != "" { capabilityArgs["deviceID"] = delegateRc.DeviceID } + if delegateRc.CNIDeviceInfoFile != "" { + capabilityArgs["CNIDeviceInfoFile"] = delegateRc.CNIDeviceInfoFile + } rt.CapabilityArgs = capabilityArgs } - return rt + return rt, cniDeviceInfoFile } // GetGatewayFromResult retrieves gateway IP addresses from CNI result @@ -311,7 +326,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { if err != nil { return nil, logging.Errorf("LoadNetConf: error marshalling delegate %d config: %v", idx, err) } - delegateConf, err := LoadDelegateNetConf(bytes, nil, "") + delegateConf, err := LoadDelegateNetConf(bytes, nil, "", "") if err != nil { return nil, logging.Errorf("LoadNetConf: failed to load delegate %d config: %v", idx, err) } diff --git a/types/conf_test.go b/types/conf_test.go index b1848b704..1caa915be 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -102,7 +102,7 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) - _, err = LoadDelegateNetConf([]byte(conf), nil, "") + _, err = LoadDelegateNetConf([]byte(conf), nil, "", "") Expect(err).To(HaveOccurred()) err = LoadDelegateNetConfList([]byte(conf), &DelegateNetConf{}) Expect(err).To(HaveOccurred()) @@ -280,7 +280,7 @@ var _ = Describe("config operations", func() { DeviceID string `json:"deviceID"` } sriovConf := &sriovNetConf{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &sriovConf) @@ -306,7 +306,7 @@ var _ = Describe("config operations", func() { Plugins []*sriovNetConf `json:"plugins"` } sriovConfList := &sriovNetConfList{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList) @@ -335,7 +335,7 @@ var _ = Describe("config operations", func() { Plugins []*sriovNetConf `json:"plugins"` } sriovConfList := &sriovNetConfList{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList) @@ -356,7 +356,7 @@ var _ = Describe("config operations", func() { PCIBusID string `json:"pciBusID"` } hostDeviceConf := &hostDeviceNetConf{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConf) @@ -382,7 +382,7 @@ var _ = Describe("config operations", func() { Plugins []*hostDeviceNetConf `json:"plugins"` } hostDeviceConfList := &hostDeviceNetConfList{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList) @@ -411,7 +411,7 @@ var _ = Describe("config operations", func() { Plugins []*hostDeviceNetConf `json:"plugins"` } hostDeviceConfList := &hostDeviceNetConfList{} - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "") Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList) @@ -444,7 +444,7 @@ var _ = Describe("config operations", func() { Name: "test-elem", CNIArgs: &args, } - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "") Expect(err).NotTo(HaveOccurred()) bridgeConf := &bridgeNetConf{} err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf) @@ -480,7 +480,7 @@ var _ = Describe("config operations", func() { Name: "test-elem", CNIArgs: &args, } - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "") Expect(err).NotTo(HaveOccurred()) bridgeConf := &bridgeNetConf{} err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf) @@ -518,7 +518,7 @@ var _ = Describe("config operations", func() { Name: "test-elem", CNIArgs: &args, } - delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "") + delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "") Expect(err).NotTo(HaveOccurred()) bridgeConflist := &bridgeNetConfList{} err = json.Unmarshal(delegateNetConf.Bytes, bridgeConflist) @@ -565,7 +565,7 @@ var _ = Describe("config operations", func() { HostIP: "anotherSampleHostIP", } - rt := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil) + rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil) fmt.Println("rt.ContainerID: ", rt.ContainerID) Expect(rt.ContainerID).To(Equal("123456789")) Expect(rt.NetNS).To(Equal(args.Netns)) @@ -595,7 +595,7 @@ var _ = Describe("config operations", func() { } }` - delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) @@ -628,7 +628,7 @@ var _ = Describe("config operations", func() { } }` - delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0") + delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "") Expect(err).NotTo(HaveOccurred()) fmt.Println("result.Version: ", result.Version()) delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil) @@ -668,7 +668,7 @@ var _ = Describe("config operations", func() { PortMappingsRequest: []*PortMapEntry{portMapEntry1}, } - delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "") + delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "", "") Expect(err).NotTo(HaveOccurred()) Expect(delegateConf.IfnameRequest).To(Equal(networkSelection.InterfaceRequest)) Expect(delegateConf.MacRequest).To(Equal(networkSelection.MacRequest)) @@ -714,7 +714,7 @@ var _ = Describe("config operations", func() { BandwidthRequest: bandwidthEntry1, PortMappingsRequest: []*PortMapEntry{portMapEntry1}, } - delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "") + delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "") delegate.MasterPlugin = true Expect(err).NotTo(HaveOccurred()) runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate) @@ -751,7 +751,7 @@ var _ = Describe("config operations", func() { BandwidthRequest: bandwidthEntry1, PortMappingsRequest: []*PortMapEntry{portMapEntry1}, } - delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "") + delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "") Expect(err).NotTo(HaveOccurred()) runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate) Expect(runtimeConf.PortMaps).NotTo(BeNil()) diff --git a/types/types.go b/types/types.go index 3a7b4cb67..8a3c668ea 100644 --- a/types/types.go +++ b/types/types.go @@ -57,12 +57,13 @@ type NetConf struct { // RuntimeConfig specifies CNI RuntimeConfig type RuntimeConfig struct { - PortMaps []*PortMapEntry `json:"portMappings,omitempty"` - Bandwidth *BandwidthEntry `json:"bandwidth,omitempty"` - IPs []string `json:"ips,omitempty"` - Mac string `json:"mac,omitempty"` - InfinibandGUID string `json:"infinibandGUID,omitempty"` - DeviceID string `json:"deviceID,omitempty"` + PortMaps []*PortMapEntry `json:"portMappings,omitempty"` + Bandwidth *BandwidthEntry `json:"bandwidth,omitempty"` + IPs []string `json:"ips,omitempty"` + Mac string `json:"mac,omitempty"` + InfinibandGUID string `json:"infinibandGUID,omitempty"` + DeviceID string `json:"deviceID,omitempty"` + CNIDeviceInfoFile string `json:"CNIDeviceInfoFile,omitempty"` } // PortMapEntry for CNI PortMapEntry @@ -111,6 +112,8 @@ type DelegateNetConf struct { ConfListPlugin bool `json:"-"` // DeviceID is only used internal housekeeping DeviceID string `json:"deviceID,omitempty"` + // ResourceName is only used internal housekeeping + ResourceName string `json:"resourceName,omitempty"` // Raw JSON Bytes []byte