rancher/multus-cni
1
0
Fork 0
forked from github/multus-cni
Code Issues Pull Requests Projects Releases Wiki Activity

Add CNIDeviceFile in RuntimeConfig

Browse Source 
DPDeviceFile is used by Device Plugins to pass device data to CNIs, as defined
in the device-info-spec. The name of the DPDeviceFile is defined by the
device-info-spec as:
 <ResourceName>-<DeviceID>-device.json

If the DPDeviceFile exists, the NPWG implementation makes a copy of the file
and passes the name of the file to the delegate CNI via capabilityArgs as
CNIDeviceFile. If the DPDeviceFile does not exist, the filename is still
passed to the CNI. The CNI can create the file and popluate it if a device
is created within the CNI.

The name of the CNIDeviceFile is not defined by device-info-spec, but to
ensure the name does not clash it is formed by the following unique triplet:
[networkName, PodUUID, ifName]

k8snetworkplumbingwg/network-attachment-definition-client repo has utility
functions to abstract some of this functionality so it can be reused across
Device Plugins, NPWG implementations and CNIs.

Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com>
This commit is contained in:
Billy McFall
2020-11-03 09:42:30 -05:00
parent fd3f485e63
commit fb557ab20f
7 changed files with 72 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
k8sclient/k8sclient.go
View File

@@ -285,7 +285,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
return nil, resourceMap, err
}
delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID)
delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID, resourceName)
if err != nil {
return nil, resourceMap, err
}
@@ -495,7 +495,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
var configBytes []byte
configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
if err == nil {
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
if err != nil {
return nil, resourceMap, err
}
@@ -514,7 +514,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
var configBytes []byte
configBytes, err = netutils.GetCNIConfigFromFile("", netname)
if err == nil {
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
if err != nil {
return nil, resourceMap, err
}

8
k8sclient/k8sclient_test.go
View File

@@ -1184,7 +1184,7 @@ users:
}
}`
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1266,7 +1266,7 @@ users:
}
}`
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1326,7 +1326,7 @@ users:
}`
// note that the provided kubeconfig is invalid
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "")
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
Expect(err).NotTo(HaveOccurred())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -1385,7 +1385,7 @@ users:
}
}`
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)

22
multus/multus.go
View File

@@ -455,11 +455,19 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
var errorstrings []string
for idx := lastIdx; idx >= 0; idx-- {
ifName := getIfname(delegates[idx], args.IfName, idx)
rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
// Attempt to delete all but do not error out, instead, collect all errors.
if err := delegateDel(exec, pod, ifName, delegates[idx], rt, binDir); err != nil {
errorstrings = append(errorstrings, err.Error())
}
if cniDeviceInfoPath != "" {
err := nadutils.CleanDeviceInfoForCNI(cniDeviceInfoPath)
// Even if the filename is set, file may not be present. Ignore error,
// but log and in the future may need to filter on specific errors.
if err != nil {
logging.Debugf("delPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err)
}
}
}
// Check if we had any errors, and send them all back.
@@ -565,8 +573,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
cniArgs := os.Getenv("CNI_ARGS")
for idx, delegate := range n.Delegates {
ifName := getIfname(delegate, args.IfName, idx)
rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
if cniDeviceInfoPath != "" {
err = nadutils.CopyDeviceInfoForCNIFromDP(cniDeviceInfoPath, delegate.ResourceName, delegate.DeviceID)
// Even if the filename is set, file may not be present. Ignore error,
// but log and in the future may need to filter on specific errors.
if err != nil {
logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
}
}
rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n.BinDir, cniArgs)
if err != nil {
// If the add failed, tear down all networks we already added
@@ -658,7 +674,7 @@ func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
for idx, delegate := range in.Delegates {
ifName := getIfname(delegate, args.IfName, idx)
rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
err = delegateCheck(exec, ifName, delegate, rt, in.BinDir)
if err != nil {
return err

4
multus/multus_test.go
View File

@@ -2031,7 +2031,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
k8sargs, err := k8sclient.GetK8sArgs(args)
n, err := types.LoadNetConf(args.StdinData)
rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
err = conflistDel(rt, rawnetconflist, binDir, fExec)
Expect(err).To(HaveOccurred())
@@ -3277,7 +3277,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`)
k8sargs, err := k8sclient.GetK8sArgs(args)
n, err := types.LoadNetConf(args.StdinData)
rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
err = conflistDel(rt, rawnetconflist, binDir, fExec)
Expect(err).To(HaveOccurred())

23
types/conf.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/containernetworking/cni/pkg/version"
nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
"gopkg.in/intel/multus-cni.v3/logging"
)
@@ -55,7 +56,7 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
}
// LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string) (*DelegateNetConf, error) {
func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
var err error
logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
@@ -87,6 +88,9 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
if err != nil {
return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConf bytes: %v", err)
}
// Save them for housekeeping
delegateConf.ResourceName = resourceName
delegateConf.DeviceID = deviceID
}
if net != nil && net.CNIArgs != nil {
bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs)
@@ -164,6 +168,7 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
if delegate.DeviceID != "" {
runtimeConfig.DeviceID = delegate.DeviceID
}
logging.Debugf("mergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", runtimeConfig)
}
return runtimeConfig
@@ -171,12 +176,19 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
// CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
// exists, merge data with the runtime config.
func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) *libcni.RuntimeConf {
func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
var cniDeviceInfoFile string
delegateRc := rc
if delegate != nil {
delegateRc = mergeCNIRuntimeConfig(delegateRc, delegate)
if delegateRc.CNIDeviceInfoFile == "" && delegate.Name != "" {
autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName)
delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo)
cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile
logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile)
}
}
// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
@@ -215,9 +227,12 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
if delegateRc.DeviceID != "" {
capabilityArgs["deviceID"] = delegateRc.DeviceID
}
if delegateRc.CNIDeviceInfoFile != "" {
capabilityArgs["CNIDeviceInfoFile"] = delegateRc.CNIDeviceInfoFile
}
rt.CapabilityArgs = capabilityArgs
}
return rt
return rt, cniDeviceInfoFile
}
// GetGatewayFromResult retrieves gateway IP addresses from CNI result
@@ -311,7 +326,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
if err != nil {
return nil, logging.Errorf("LoadNetConf: error marshalling delegate %d config: %v", idx, err)
}
delegateConf, err := LoadDelegateNetConf(bytes, nil, "")
delegateConf, err := LoadDelegateNetConf(bytes, nil, "", "")
if err != nil {
return nil, logging.Errorf("LoadNetConf: failed to load delegate %d config: %v", idx, err)
}

32
types/conf_test.go
View File

@@ -102,7 +102,7 @@ var _ = Describe("config operations", func() {
_, err := LoadNetConf([]byte(conf))
Expect(err).To(HaveOccurred())
_, err = LoadDelegateNetConf([]byte(conf), nil, "")
_, err = LoadDelegateNetConf([]byte(conf), nil, "", "")
Expect(err).To(HaveOccurred())
err = LoadDelegateNetConfList([]byte(conf), &DelegateNetConf{})
Expect(err).To(HaveOccurred())
@@ -280,7 +280,7 @@ var _ = Describe("config operations", func() {
DeviceID string `json:"deviceID"`
}
sriovConf := &sriovNetConf{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &sriovConf)
@@ -306,7 +306,7 @@ var _ = Describe("config operations", func() {
Plugins []*sriovNetConf `json:"plugins"`
}
sriovConfList := &sriovNetConfList{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -335,7 +335,7 @@ var _ = Describe("config operations", func() {
Plugins []*sriovNetConf `json:"plugins"`
}
sriovConfList := &sriovNetConfList{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -356,7 +356,7 @@ var _ = Describe("config operations", func() {
PCIBusID string `json:"pciBusID"`
}
hostDeviceConf := &hostDeviceNetConf{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConf)
@@ -382,7 +382,7 @@ var _ = Describe("config operations", func() {
Plugins []*hostDeviceNetConf `json:"plugins"`
}
hostDeviceConfList := &hostDeviceNetConfList{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -411,7 +411,7 @@ var _ = Describe("config operations", func() {
Plugins []*hostDeviceNetConf `json:"plugins"`
}
hostDeviceConfList := &hostDeviceNetConfList{}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
Expect(err).NotTo(HaveOccurred())
err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -444,7 +444,7 @@ var _ = Describe("config operations", func() {
Name: "test-elem",
CNIArgs: &args,
}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
Expect(err).NotTo(HaveOccurred())
bridgeConf := &bridgeNetConf{}
err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -480,7 +480,7 @@ var _ = Describe("config operations", func() {
Name: "test-elem",
CNIArgs: &args,
}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
Expect(err).NotTo(HaveOccurred())
bridgeConf := &bridgeNetConf{}
err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -518,7 +518,7 @@ var _ = Describe("config operations", func() {
Name: "test-elem",
CNIArgs: &args,
}
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
Expect(err).NotTo(HaveOccurred())
bridgeConflist := &bridgeNetConfList{}
err = json.Unmarshal(delegateNetConf.Bytes, bridgeConflist)
@@ -565,7 +565,7 @@ var _ = Describe("config operations", func() {
HostIP: "anotherSampleHostIP",
}
rt := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil)
rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil)
fmt.Println("rt.ContainerID: ", rt.ContainerID)
Expect(rt.ContainerID).To(Equal("123456789"))
Expect(rt.NetNS).To(Equal(args.Netns))
@@ -595,7 +595,7 @@ var _ = Describe("config operations", func() {
}
}`
delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -628,7 +628,7 @@ var _ = Describe("config operations", func() {
}
}`
delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
Expect(err).NotTo(HaveOccurred())
fmt.Println("result.Version: ", result.Version())
delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
@@ -668,7 +668,7 @@ var _ = Describe("config operations", func() {
PortMappingsRequest: []*PortMapEntry{portMapEntry1},
}
delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "")
delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "", "")
Expect(err).NotTo(HaveOccurred())
Expect(delegateConf.IfnameRequest).To(Equal(networkSelection.InterfaceRequest))
Expect(delegateConf.MacRequest).To(Equal(networkSelection.MacRequest))
@@ -714,7 +714,7 @@ var _ = Describe("config operations", func() {
BandwidthRequest: bandwidthEntry1,
PortMappingsRequest: []*PortMapEntry{portMapEntry1},
}
delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
delegate.MasterPlugin = true
Expect(err).NotTo(HaveOccurred())
runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
@@ -751,7 +751,7 @@ var _ = Describe("config operations", func() {
BandwidthRequest: bandwidthEntry1,
PortMappingsRequest: []*PortMapEntry{portMapEntry1},
}
delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
Expect(err).NotTo(HaveOccurred())
runtimeConf := mergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
Expect(runtimeConf.PortMaps).NotTo(BeNil())

3
types/types.go
View File

@@ -63,6 +63,7 @@ type RuntimeConfig struct {
Mac string `json:"mac,omitempty"`
InfinibandGUID string `json:"infinibandGUID,omitempty"`
DeviceID string `json:"deviceID,omitempty"`
CNIDeviceInfoFile string `json:"CNIDeviceInfoFile,omitempty"`
}
// PortMapEntry for CNI PortMapEntry
@@ -111,6 +112,8 @@ type DelegateNetConf struct {
ConfListPlugin bool `json:"-"`
// DeviceID is only used internal housekeeping
DeviceID string `json:"deviceID,omitempty"`
// ResourceName is only used internal housekeeping
ResourceName string `json:"resourceName,omitempty"`
// Raw JSON
Bytes []byte