Merge pull request #1470 from trozet/add_status_delegation

Adds support for CNI STATUS + other fixes for CNI Spec 1.1.0

cmd/thin_entrypoint/main.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/libcni"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/spf13/pflag"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
@@ -497,14 +498,14 @@ func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, [
 		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
 	}
 
-	// use conflist template if cniVersionConfig == "1.0.0"
+	// use conflist template if cniVersionConfig >= "1.0.0"
 	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
 	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
 	if err != nil {
 		return "", nil, fmt.Errorf("template parse error: %v", err)
 	}
 
-	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
+	if gt, err := cniversion.GreaterThanOrEqualTo(o.CNIVersion, "1.0.0"); err == nil && gt {
 		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
 		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
 		if err != nil {

cmd/thin_entrypoint/main_test.go

@@ -318,6 +318,56 @@ var _ = Describe("thin entrypoint testing", func() {
 		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
+	It("Run createMultusConfig(), default, conflist for cniVersion 1.1.0", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.1.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.1.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.1.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
 	It("Run createMultusConfig(), capabilities, conflist", func() {
 		// create directory and files
 		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")

pkg/multus/multus.go

@@ -18,6 +18,7 @@ package multus
 import (
 	"context"
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"net"
 	"os"
@@ -30,6 +31,7 @@ import (
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
@@ -258,6 +260,42 @@ func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.Net
 	return err
 }
 
+func confStatus(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("confStatus: %v, %s", rt, string(rawNetconf))
+
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
+
+	conf, err := libcni.ConfFromBytes(rawNetconf)
+	if err != nil {
+		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	}
+
+	if gt, _ := cniversion.GreaterThanOrEqualTo(conf.Network.CNIVersion, "1.1.0"); !gt {
+		logging.Debugf("confStatus: skipping STATUS for network %q type %q cniVersion %q (< 1.1.0)",
+			conf.Network.Name, conf.Network.Type, conf.Network.CNIVersion)
+		return nil
+	}
+
+	confList := &libcni.NetworkConfigList{
+		Name:       conf.Network.Name,
+		CNIVersion: conf.Network.CNIVersion,
+		Plugins:    []*libcni.PluginConfig{conf},
+	}
+
+	err = cniNet.GetStatusNetworkList(context.Background(), confList)
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		return logging.Errorf("error in getting result from StatusNetworkList: %v", err)
+	}
+
+	return err
+}
+
 func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, cniConfList *libcni.NetworkConfigList, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
 	logging.Debugf("conflistAdd: %v, %s", rt, string(rawnetconflist))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
@@ -327,6 +365,33 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *t
 	return err
 }
 
+func conflistStatus(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("conflistStatus: %v, %s", rt, string(rawnetconflist))
+
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
+
+	confList, err := libcni.ConfListFromBytes(rawnetconflist)
+	if err != nil {
+		return logging.Errorf("conflistStatus: error converting the raw bytes into a conflist: %v", err)
+	}
+	if gt, _ := cniversion.GreaterThanOrEqualTo(confList.CNIVersion, "1.1.0"); !gt {
+		logging.Debugf("conflistStatus: skipping STATUS for network list %q cniVersion %q (< 1.1.0)", confList.Name, confList.CNIVersion)
+	}
+
+	err = cniNet.GetStatusNetworkList(context.Background(), confList)
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		return logging.Errorf("conflistStatus: error in getting result from StatusNetworkList: %v", err)
+	}
+
+	return err
+}
+
 // DelegateAdd ...
 func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
 	logging.Debugf("DelegateAdd: %v, %v, %v", exec, delegate, rt)
@@ -456,6 +521,46 @@ func DelegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *li
 	return err
 }
 
+// DelegateStatus ...
+func DelegateStatus(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("DelegateStatus: %v, %v, %v", exec, delegateConf, rt)
+
+	isConfList := delegateConf.ConfListPlugin
+	if !isConfList && delegateConf.Conf.Type == "" && delegateConf.ConfList.Name != "" {
+		isConfList = true
+	}
+
+	if logging.GetLoggingLevel() >= logging.VerboseLevel {
+		var cniConfName string
+		if isConfList {
+			cniConfName = delegateConf.ConfList.Name
+		} else {
+			cniConfName = delegateConf.Conf.Name
+		}
+		logging.Verbosef("Status: %s:%s:%s(%s):%s %s", rt.Args[1][1], rt.Args[2][1], delegateConf.Name, cniConfName, rt.IfName, string(delegateConf.Bytes))
+	}
+
+	var err error
+	if isConfList {
+		err = conflistStatus(rt, delegateConf.Bytes, multusNetconf, exec)
+	} else {
+		err = confStatus(rt, delegateConf.Bytes, multusNetconf, exec)
+	}
+
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		if isConfList {
+			return logging.Errorf("DelegateStatus: error invoking ConflistStatus - %q: %v", delegateConf.ConfList.Name, err)
+		}
+		return logging.Errorf("DelegateStatus: error invoking ConfStatus - %q: %v", delegateConf.Conf.Type, err)
+	}
+
+	return err
+}
+
 // DelegateDel ...
 func DelegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
 	logging.Debugf("DelegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
@@ -659,10 +764,10 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 	pod, err := GetPod(kubeClient, k8sArgs, false)
 	if err != nil {
-		if err == errPodNotFound {
-			emptyresult := emptyCNIResult(args, "1.0.0")
-			logging.Verbosef("CmdAdd: Warning: pod [%s/%s] not found, exiting with empty CNI result: %v", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, emptyresult)
-			return emptyresult, nil
+		if stderrors.Is(err, errPodNotFound) {
+			emptyResult := emptyCNIResult(args, n.CNIVersion)
+			logging.Verbosef("CmdAdd: Warning: pod [%s/%s] not found, exiting with empty CNI result: %v", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, emptyResult)
+			return emptyResult, nil
 		}
 		return nil, err
 	}
@@ -1050,17 +1155,26 @@ func CmdStatus(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 
 	// invoke delegate's STATUS command
 	// we only need to check cluster network status
+	delegate := n.Delegates[0]
+	if !delegate.ConfListPlugin {
+		return confStatus(&libcni.RuntimeConf{}, delegate.Bytes, n, exec)
+	}
+
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
 	binDirs = append([]string{n.BinDir}, binDirs...)
 	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
 
-	conf, err := libcni.ConfListFromBytes(n.Delegates[0].Bytes)
+	conf, err := libcni.ConfListFromBytes(delegate.Bytes)
 	if err != nil {
 		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
 	}
 
-	err = cniNet.GetStatusNetworkList(context.TODO(), conf)
+	err = cniNet.GetStatusNetworkList(context.Background(), conf)
 	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
 		return logging.Errorf("error in STATUS command: %v", err)
 	}
 
@@ -1101,9 +1215,28 @@ func CmdGC(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) err
 	binDirs = append([]string{n.BinDir}, binDirs...)
 	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
 
-	conf, err := libcni.ConfListFromBytes(n.Delegates[0].Bytes)
-	if err != nil {
-		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	delegate := n.Delegates[0]
+	isConfList := delegate.ConfListPlugin
+	if !isConfList && delegate.Conf.Type == "" && delegate.ConfList.Name != "" {
+		isConfList = true
+	}
+
+	var confList *libcni.NetworkConfigList
+	if isConfList {
+		confList, err = libcni.ConfListFromBytes(delegate.Bytes)
+		if err != nil {
+			return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+		}
+	} else {
+		conf, err := libcni.ConfFromBytes(delegate.Bytes)
+		if err != nil {
+			return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+		}
+		confList = &libcni.NetworkConfigList{
+			Name:       conf.Network.Name,
+			CNIVersion: conf.Network.CNIVersion,
+			Plugins:    []*libcni.PluginConfig{conf},
+		}
 	}
 
 	validAttachments, err := gatherValidAttachmentsFromCache(n.CNIDir)
@@ -1111,7 +1244,7 @@ func CmdGC(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) err
 		return logging.Errorf("error in gather valid attachments: %v", err)
 	}
 
-	err = cniNet.GCNetworkList(context.TODO(), conf, &libcni.GCArgs{
+	err = cniNet.GCNetworkList(context.TODO(), confList, &libcni.GCArgs{
 		ValidAttachments: validAttachments,
 	})
 	if err != nil {

pkg/multus/multus_cni100_test.go

@@ -18,6 +18,7 @@ package multus
 //revive:disable:dot-imports
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -26,11 +27,13 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -1244,6 +1247,109 @@ var _ = Describe("multus operations cniVersion 1.1.0 config", func() {
 		Expect(fExec.statusIndex).To(Equal(1))
 	})
 
+	It("returns empty add result using top-level cniVersion when pod is not found", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        "K8S_POD_NAME=missing-pod;K8S_POD_NAMESPACE=default",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniVersion": "1.1.0",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fKubeClient := NewFakeClientInfo()
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		r, ok := result.(*cni100.Result)
+		Expect(ok).To(BeTrue())
+		Expect(r.CNIVersion).To(Equal("1.1.0"))
+		Expect(fExec.addIndex).To(Equal(0))
+	})
+
+	It("propagates delegate STATUS errors", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+		}
+		k8sArgs := &types.K8sArgs{
+			K8S_POD_NAMESPACE:          cnitypes.UnmarshallableString("default"),
+			K8S_POD_NAME:               cnitypes.UnmarshallableString("pod"),
+			K8S_POD_INFRA_CONTAINER_ID: cnitypes.UnmarshallableString("sandbox"),
+			K8S_POD_UID:                cnitypes.UnmarshallableString("uid"),
+		}
+
+		delegateConf, err := types.LoadDelegateNetConf([]byte(`{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`), nil, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, args.IfName, nil, delegateConf)
+
+		fExec := newFakeExec()
+		expectedConf := `{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "", expectedConf, nil, &cnitypes.Error{Code: 50, Msg: "status failed"})
+
+		err = DelegateStatus(fExec, delegateConf, rt, &types.NetConf{BinDir: "/bin", CNIDir: tmpDir})
+		Expect(err).To(HaveOccurred())
+		var cniErr *cnitypes.Error
+		Expect(errors.As(err, &cniErr)).To(BeTrue())
+		Expect(cniErr.Code).To(Equal(uint(50)))
+		Expect(cniErr.Msg).To(Equal("status failed"))
+	})
+
+	It("propagates CmdStatus errors for single plugin delegates", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedConf := `{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "", expectedConf, nil, &cnitypes.Error{Code: 50, Msg: "status failed"})
+
+		err := CmdStatus(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+		var cniErr *cnitypes.Error
+		Expect(errors.As(err, &cniErr)).To(BeTrue())
+		Expect(cniErr.Code).To(Equal(uint(50)))
+		Expect(cniErr.Msg).To(Equal("status failed"))
+	})
+
 	It("executes delegates with CNI GC", func() {
 		tmpCNIDir := tmpDir + "/cniData"
 		err := os.Mkdir(tmpCNIDir, 0777)
@@ -1322,4 +1428,42 @@ var _ = Describe("multus operations cniVersion 1.1.0 config", func() {
 		err = os.RemoveAll(tmpCNIDir)
 		Expect(err).NotTo(HaveOccurred())
 	})
+
+	It("executes single plugin delegates with CNI GC", func() {
+		tmpCNIDir := tmpDir + "/cniData-single"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		cniCacheDir := filepath.Join(tmpCNIDir, "/results")
+		err = os.Mkdir(cniCacheDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "", "", nil, nil)
+
+		err = CmdGC(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.gcIndex).To(Equal(1))
+
+		err = os.RemoveAll(tmpCNIDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
 })

pkg/netutils/netutils.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 
 	"github.com/containernetworking/cni/libcni"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
@@ -183,7 +184,7 @@ func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[
 		return deleteDefaultGWResult020(result, ipv4, ipv6)
 	}
 
-	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+	if !isSupportedGatewayResultVersion(cniVersion) {
 		return nil, fmt.Errorf("not supported version: %s", cniVersion)
 	}
 
@@ -340,7 +341,7 @@ func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]
 		return addDefaultGWResult020(result, gw)
 	}
 
-	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+	if !isSupportedGatewayResultVersion(cniVersion) {
 		return nil, fmt.Errorf("not supported version: %s", cniVersion)
 	}
 
@@ -368,6 +369,19 @@ func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]
 	return result, nil
 }
 
+func isSupportedGatewayResultVersion(cniVersion string) bool {
+	switch cniVersion {
+	case "0.3.0", "0.3.1", "0.4.0":
+		return true
+	}
+
+	if gt, _ := cniversion.GreaterThanOrEqualTo(cniVersion, "1.0.0"); gt {
+		return true
+	}
+
+	return false
+}
+
 func addDefaultGWResult020(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
 	for _, g := range gw {
 		if g.To4() != nil {

pkg/netutils/netutils_test.go

@@ -1508,4 +1508,34 @@ var _ = Describe("other function unit testing", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(routeJSON).Should(MatchJSON(`[{"dst":"10.1.1.0/24"}]`))
 	})
+
+	It("supports gateway result updates for cniVersion 1.1.0", func() {
+		deleteInput := map[string]interface{}{
+			"cniVersion": "1.1.0",
+			"routes": []interface{}{
+				map[string]interface{}{"dst": "0.0.0.0/0", "gw": "10.1.1.1"},
+			},
+		}
+		updatedDeleteResult, err := deleteDefaultGWResult(deleteInput, true, false)
+		Expect(err).NotTo(HaveOccurred())
+		_, hasRoutes := updatedDeleteResult["routes"]
+		Expect(hasRoutes).To(BeFalse())
+
+		addInput := map[string]interface{}{
+			"cniVersion": "1.1.0",
+		}
+		updatedAddResult, err := addDefaultGWResult(addInput, []net.IP{net.ParseIP("10.1.1.1")})
+		Expect(err).NotTo(HaveOccurred())
+		routes, ok := updatedAddResult["routes"].([]interface{})
+		Expect(ok).To(BeTrue())
+		Expect(routes).To(HaveLen(1))
+	})
+
+	It("rejects unsupported pre-1.0.0 cniVersion", func() {
+		addInput := map[string]interface{}{
+			"cniVersion": "0.9.0",
+		}
+		_, err := addDefaultGWResult(addInput, []net.IP{net.ParseIP("10.1.1.1")})
+		Expect(err).To(MatchError("not supported version: 0.9.0"))
+	})
 })

pkg/server/api/api.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 	"time"
 
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -71,6 +73,10 @@ func DoCNI(url string, req interface{}, socketPath string) ([]byte, error) {
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		cniErr := &cnitypes.Error{}
+		if err := json.Unmarshal(body, cniErr); err == nil && cniErr.Msg != "" {
+			return nil, cniErr
+		}
 		return nil, fmt.Errorf("CNI request failed with status %v: '%s'", resp.StatusCode, string(body))
 	}
 

pkg/server/api/shim.go

@@ -16,6 +16,7 @@ package api
 
 import (
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"os"
 	"strings"
@@ -111,6 +112,10 @@ func postRequest(args *skel.CmdArgs, readinessCheck readyCheckFunc) (*Response,
 	var body []byte
 	body, err = DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
 	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return nil, multusShimConfig.CNIVersion, err
+		}
 		return nil, multusShimConfig.CNIVersion, fmt.Errorf("%s: StdinData: %s", err.Error(), string(args.StdinData))
 	}
 

pkg/server/server.go

@@ -17,6 +17,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -125,6 +126,8 @@ func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCm
 		err = s.cmdDelegateDel(cniCmdArgs, k8sArgs, multusConfig)
 	case "CHECK":
 		err = s.cmdDelegateCheck(cniCmdArgs, k8sArgs, multusConfig)
+	case "STATUS":
+		err = s.cmdDelegateStatus(cniCmdArgs, k8sArgs, multusConfig)
 	default:
 		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
 	}
@@ -302,7 +305,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 
 			result, err := s.handleCNIRequest(r)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				s.writeCNIErrorResponse(w, err)
 				return
 			}
 
@@ -324,7 +327,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 
 			result, err := s.handleDelegateRequest(r)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				s.writeCNIErrorResponse(w, err)
 				return
 			}
 
@@ -389,6 +392,34 @@ func (s *Server) Start(ctx context.Context, l net.Listener) {
 	}()
 }
 
+func (s *Server) writeCNIErrorResponse(w http.ResponseWriter, err error) {
+	var cniErr *cnitypes.Error
+	if errors.As(err, &cniErr) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		errBytes, marshalErr := json.Marshal(cniErr)
+		if marshalErr != nil {
+			http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+			return
+		}
+		if _, writeErr := w.Write(errBytes); writeErr != nil {
+			_ = logging.Errorf("Error writing HTTP response: %v", writeErr)
+		}
+		return
+	}
+	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+}
+
+func (s *Server) wrapCNIRequestError(cmdArgs *skel.CmdArgs, err error) error {
+	var cniErr *cnitypes.Error
+	if errors.As(err, &cniErr) {
+		_ = logging.Errorf("%s ERRORED: %v", printCmdArgs(cmdArgs), err)
+		return err
+	}
+	// Prefix error with request information for easier debugging.
+	return fmt.Errorf("%s ERRORED: %v", printCmdArgs(cmdArgs), err)
+}
+
 func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 	var cr api.Request
 	b, err := io.ReadAll(r.Body)
@@ -410,8 +441,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 
 	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs)
 	if err != nil {
-		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%s ERRORED: %v", printCmdArgs(cniCmdArgs), err)
+		return nil, s.wrapCNIRequestError(cniCmdArgs, err)
 	}
 	return result, nil
 }
@@ -437,8 +467,7 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 
 	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, cr.InterfaceAttributes)
 	if err != nil {
-		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%s ERRORED: %v", printCmdArgs(cniCmdArgs), err)
+		return nil, s.wrapCNIRequestError(cniCmdArgs, err)
 	}
 	return result, nil
 }
@@ -705,6 +734,15 @@ func (s *Server) cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs,
 	return multus.DelegateCheck(s.exec, delegateCNIConf, rt, multusConfig)
 }
 
+func (s *Server) cmdDelegateStatus(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
+	delegateCNIConf, err := types.LoadDelegateNetConf(cmdArgs.StdinData, nil, "", "")
+	if err != nil {
+		return err
+	}
+	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
+	return multus.DelegateStatus(s.exec, delegateCNIConf, rt, multusConfig)
+}
+
 // note: this function may send back error to the client. In cni spec, command DEL should NOT send any error
 // because container deletion follows cni DEL command. But in delegateDel case, container is not removed by
 // this delegateDel, hence we decide to send error message to the request sender.