Merge pull request #828 from s1061123/dev/chroot

Add chroot option in multus-daemon

cmd/multus-daemon/main.go

@@ -49,7 +49,7 @@ const (
 	defaultMultusMasterCNIFile          = ""
 	defaultMultusNamespaceIsolation     = false
 	defaultMultusReadinessIndicatorFile = ""
-	defaultMultusRunDir                 = "/host/var/run/multus-cni/"
+	defaultMultusRunDir                 = "/host/run/multus-cni/"
 )
 
 const (
@@ -228,7 +228,7 @@ func startMultusDaemon(configFilePath string) error {
 		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
 	}
 
-	server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir, config)
+	server, err := srv.NewCNIServer(daemonConfig, config)
 	if err != nil {
 		return fmt.Errorf("failed to create the server: %v", err)
 	}

deployments/multus-daemonset-thick.yml

@@ -102,13 +102,14 @@ metadata:
 data:
   daemon-config.json: |
     {
+        "chrootDir": "/hostroot",
         "confDir": "/host/etc/cni/net.d",
         "logToStderr": true,
         "logLevel": "debug",
         "logFile": "/tmp/multus.log",
-        "binDir": "/host/opt/cni/bin",
+        "binDir": "/opt/cni/bin",
-        "cniDir": "/host/var/lib/cni/multus",
+        "cniDir": "/var/lib/cni/multus",
-        "socketDir": "/host/var/run/multus/"
+        "socketDir": "/host/run/multus/"
     }
 ---
 apiVersion: apps/v1
@@ -163,16 +164,19 @@ spec:
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
-            - name: cnibin
+            - name: host-run
-              mountPath: /host/opt/cni/bin
+              mountPath: /host/run
-            - name: host-var-run
+            - name: host-var-lib-cni-multus
-              mountPath: /host/var/run
+              mountPath: /var/lib/cni/multus
-            - name: host-var-run-netns
+            - name: host-run-netns
-              mountPath: /var/run/netns
+              mountPath: /run/netns
               mountPropagation: HostToContainer
             - name: multus-daemon-config
               mountPath: /etc/cni/net.d/multus.d
               readOnly: true
+            - name: hostroot
+              mountPath: /hostroot
+              mountPropagation: HostToContainer
       initContainers:
         - name: install-multus-binary
           image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
@@ -198,15 +202,21 @@ spec:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
+        - name: hostroot
+          hostPath:
+            path: /
         - name: multus-daemon-config
           configMap:
             name: multus-daemon-config
             items:
             - key: daemon-config.json
               path: daemon-config.json
-        - name: host-var-run
+        - name: host-run
           hostPath:
-            path: /var/run
+            path: /run
-        - name: host-var-run-netns
+        - name: host-var-lib-cni-multus
           hostPath:
-            path: /var/run/netns/
+            path: /var/lib/cni/multus
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

docs/how-to-use.md

@@ -39,7 +39,7 @@ cat >/etc/cni/net.d/00-multus.conf <<EOF
 {
   "name": "multus-cni-network",
   "type": "multus",
-  "readinessindicatorfile": "/var/run/flannel/subnet.env",
+  "readinessindicatorfile": "/run/flannel/subnet.env",
   "delegates": [
     {
       "NOTE1": "This is example, wrote your CNI config in delegates",

docs/thick-plugin.md

@@ -79,7 +79,14 @@ specifies the path to the server configuration:
 
 The server configuration is encoded in JSON, and allows the following keys:
 
+- `"chrootDir"`: Specify the directory which points to host root from the pod. See 'Chroot configuration' section for the details.
 - `"socketDir"`: Specify the location where the unix domain socket used for
-client/server communication will be located. Defaults to `"/var/run/multus-cni/"`.
+client/server communication will be located. Defaults to `"/run/multus-cni"`.
 
 In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
+
+#### Chroot configuration
+
+In thick plugin case, delegate CNI plugin is executed by multus-daemon from Pod, hence if the delegate CNI requires resources in container host, for example unix socket or even file, then CNI plugin is failed to execute because multus-daemon runs in Pod. Multus-daemon supports "chrootDir" option which executes delegate CNI under chroot (to container host).
+
+This configuration is enabled in deployments/multus-daemonset-thick.yml as default.

e2e/templates/multus-daemonset-thick.yml.j2

@@ -89,8 +89,8 @@ data:
         "logLevel": "debug",
         "logFile": "/tmp/multus.log",
         "binDir": "/host/opt/cni/bin",
-        "cniDir": "/host/var/lib/cni/multus",
+        "cniDir": "/var/lib/cni/multus",
-        "socketDir": "/host/var/run/multus/"
+        "socketDir": "/host/run/multus"
     }
 ---
 apiVersion: apps/v1
@@ -148,10 +148,12 @@ spec:
           mountPath: /host/etc/cni/net.d
         - name: cnibin
           mountPath: /host/opt/cni/bin
-        - name: host-var-run
+        - name: host-run
-          mountPath: /host/var/run
+          mountPath: /host/run
-        - name: host-var-run-netns
+        - name: host-var-lib-cni-multus
-          mountPath: /var/run/netns
+          mountPath: /var/lib/cni/multus
+        - name: host-run-netns
+          mountPath: /run/netns
           mountPropagation: HostToContainer
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
@@ -186,9 +188,12 @@ spec:
             items:
             - key: daemon-config.json
               path: daemon-config.json
-        - name: host-var-run
+        - name: host-run
           hostPath:
-            path: /var/run
+            path: /run
-        - name: host-var-run-netns
+        - name: host-var-lib-cni-multus
           hostPath:
-            path: /var/run/netns/
+            path: /var/lib/cni/multus
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

images/entrypoint.sh

@@ -223,7 +223,7 @@ MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
 
 # ------------------------------- Generate a "kube-config"
 # Inspired by: https://tinyurl.com/y7r2knme
-SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+SERVICE_ACCOUNT_PATH=/run/secrets/kubernetes.io/serviceaccount
 KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
 SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
 SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}

pkg/k8sclient/k8sclient.go

@@ -578,16 +578,16 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 			if strings.HasSuffix(netname, ".conflist") {
 				confList, err := libcni.ConfListFromFile(netname)
 				if err != nil {
-					return nil, resourceMap, fmt.Errorf("Error loading CNI conflist file %s: %v", netname, err)
+					return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
 				}
 				configBytes = confList.Bytes
 			} else {
 				conf, err := libcni.ConfFromFile(netname)
 				if err != nil {
-					return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: %v", netname, err)
+					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
 				}
 				if conf.Network.Type == "" {
-					return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
+					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
 				}
 				configBytes = conf.Bytes
 			}

pkg/netutils/netutils_test.go

@@ -429,7 +429,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -503,7 +503,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -577,7 +577,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -649,7 +649,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -973,7 +973,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1021,7 +1021,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1087,7 +1087,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1135,7 +1135,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1205,7 +1205,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1251,7 +1251,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1315,7 +1315,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1361,7 +1361,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [

pkg/server/exec_chroot.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containernetworking/cni/pkg/invoke"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/version"
+)
+
+// ChrootExec implements invoke.Exec to execute CNI with chroot
+type ChrootExec struct {
+	Stderr     io.Writer
+	chrootDir  string
+	workingDir string   // working directory in the outer root
+	outerRoot  *os.File // outer root directory
+	version.PluginDecoder
+	mu sync.Mutex
+}
+
+var _ invoke.Exec = &ChrootExec{}
+
+func (e *ChrootExec) chroot() error {
+	var err error
+	e.workingDir, err = os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
+		return fmt.Errorf("getwd before chroot failed: %v", err)
+	}
+
+	e.outerRoot, err = os.Open("/")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
+		return fmt.Errorf("getwd before chroot failed: %v", err)
+	}
+
+	if err := syscall.Chroot(e.chrootDir); err != nil {
+		fmt.Fprintf(os.Stderr, "chroot to %s failed: %v\n", e.chrootDir, err)
+		return fmt.Errorf("chroot to %s failed: %v", e.chrootDir, err)
+	}
+
+	if err := os.Chdir("/"); err != nil {
+		fmt.Fprintf(os.Stderr, "chdir to \"/\" failed: %v\n", err)
+		return fmt.Errorf("chdir to \"/\" failed: %v", err)
+	}
+
+	return nil
+}
+
+func (e *ChrootExec) escape() error {
+	if e.outerRoot == nil || e.workingDir == "" {
+		return nil
+	}
+
+	// change directory to outer root and close it
+	if err := syscall.Fchdir(int(e.outerRoot.Fd())); err != nil {
+		fmt.Fprintf(os.Stderr, "changing directory to outer root failed: %v\n", err)
+		return fmt.Errorf("changing directory to outer root failed: %v", err)
+	}
+
+	if err := e.outerRoot.Close(); err != nil {
+		fmt.Fprintf(os.Stderr, "closing outer root failed: %v\n", err)
+		return fmt.Errorf("closing outer root failed: %v", err)
+	}
+
+	// chroot to current directory aka "." being the outer root
+	if err := syscall.Chroot("."); err != nil {
+		fmt.Fprintf(os.Stderr, "chroot to current directory failed: %v\n", err)
+		return fmt.Errorf("chroot to current directory failed: %v", err)
+	}
+
+	if err := os.Chdir(e.workingDir); err != nil {
+		fmt.Fprintf(os.Stderr, "chdir to working directory failed: %v\n", err)
+		return fmt.Errorf("chdir to working directory failed: %v", err)
+	}
+	e.outerRoot = nil
+	e.workingDir = ""
+
+	return nil
+}
+
+// ExecPlugin executes CNI plugin with given environment/stdin data.
+func (e *ChrootExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
+	// lock and do chroot to execute plugin with host root
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	err := e.chroot()
+	defer e.escape()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ExecPlugin failed at chroot: %v\n", err)
+		return nil, fmt.Errorf("ExecPlugin failed at chroot: %v", err)
+	}
+
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	c := exec.CommandContext(ctx, pluginPath)
+	c.Env = environ
+	c.Stdin = bytes.NewBuffer(stdinData)
+	c.Stdout = stdout
+	c.Stderr = stderr
+
+	// Retry the command on "text file busy" errors
+	for i := 0; i <= 5; i++ {
+		err = c.Run()
+
+		// Command succeeded
+		if err == nil {
+			break
+		}
+
+		// If the plugin is currently about to be written, then we wait a
+		// second and try it again
+		if strings.Contains(err.Error(), "text file busy") {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// All other errors except than the busy text file
+		return nil, e.pluginErr(err, stdout.Bytes(), stderr.Bytes())
+	}
+
+	// Copy stderr to caller's buffer in case plugin printed to both
+	// stdout and stderr for some reason. Ignore failures as stderr is
+	// only informational.
+	if e.Stderr != nil && stderr.Len() > 0 {
+		_, _ = stderr.WriteTo(e.Stderr)
+	}
+	return stdout.Bytes(), nil
+}
+
+func (e *ChrootExec) pluginErr(err error, stdout, stderr []byte) error {
+	emsg := types.Error{}
+	if len(stdout) == 0 {
+		if len(stderr) == 0 {
+			emsg.Msg = fmt.Sprintf("netplugin failed with no error message: %v", err)
+		} else {
+			emsg.Msg = fmt.Sprintf("netplugin failed: %q", string(stderr))
+		}
+	} else if perr := json.Unmarshal(stdout, &emsg); perr != nil {
+		emsg.Msg = fmt.Sprintf("netplugin failed but error parsing its diagnostic message %q: %v", string(stdout), perr)
+	}
+	return &emsg
+}
+
+// FindInPath try to find CNI plugin based on given path
+func (e *ChrootExec) FindInPath(plugin string, paths []string) (string, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	err := e.chroot()
+	defer e.escape()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "FindInPath failed at chroot: %v\n", err)
+		return "", fmt.Errorf("FindInPath failed at chroot: %v", err)
+	}
+
+	return invoke.FindInPath(plugin, paths)
+}

pkg/server/server.go

@@ -77,13 +77,22 @@ func GetListener(socketPath string) (net.Listener, error) {
 }
 
 // NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
-func NewCNIServer(rundir string, serverConfig []byte) (*Server, error) {
+func NewCNIServer(daemonConfig *types.ControllerNetConf, serverConfig []byte) (*Server, error) {
 	kubeClient, err := k8s.InClusterK8sClient()
 	if err != nil {
 		return nil, fmt.Errorf("error getting k8s client: %v", err)
 	}
 
-	return newCNIServer(rundir, kubeClient, nil, serverConfig)
+	exec := invoke.Exec(nil)
+	if daemonConfig.ChrootDir != "" {
+		exec = &ChrootExec{
+			Stderr:    os.Stderr,
+			chrootDir: daemonConfig.ChrootDir,
+		}
+		logging.Verbosef("server configured with chroot: %s", daemonConfig.ChrootDir)
+	}
+
+	return newCNIServer(daemonConfig.MultusSocketDir, kubeClient, exec, serverConfig)
 }
 
 func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {

pkg/server/shim.go

@@ -18,13 +18,14 @@ import (
 )
 
 const (
-	defaultMultusRunDir = "/var/run/multus/"
+	defaultMultusRunDir = "/run/multus/"
 )
 
 // CmdAdd implements the CNI spec ADD command handler
 func CmdAdd(args *skel.CmdArgs) error {
 	response, cniVersion, err := postRequest(args)
 	if err != nil {
+		logging.Errorf("CmdAdd (shim): %v", err)
 		return err
 	}
 
@@ -34,24 +35,24 @@ func CmdAdd(args *skel.CmdArgs) error {
 
 // CmdCheck implements the CNI spec CHECK command handler
 func CmdCheck(args *skel.CmdArgs) error {
-	response, cniVersion, err := postRequest(args)
+	_, _, err := postRequest(args)
 	if err != nil {
+		logging.Errorf("CmdCheck (shim): %v", err)
 		return err
 	}
 
-	logging.Verbosef("CmdCheck (shim): %v", *response.Result)
+	return err
-	return cnitypes.PrintResult(response.Result, cniVersion)
 }
 
 // CmdDel implements the CNI spec DEL command handler
 func CmdDel(args *skel.CmdArgs) error {
-	response, cniVersion, err := postRequest(args)
+	_, _, err := postRequest(args)
 	if err != nil {
-		return err
+		logging.Errorf("CmdDel (shim): %v", err)
+		return nil
 	}
 
-	logging.Verbosef("CmdDel (shim): %v", *response.Result)
+	return nil
-	return cnitypes.PrintResult(response.Result, cniVersion)
 }
 
 func postRequest(args *skel.CmdArgs) (*Response, string, error) {

pkg/types/conf.go

@@ -44,7 +44,7 @@ const (
 const (
 	// DefaultMultusDaemonConfigFile is the default path of the config file
 	DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json"
-	defaultMultusRunDir           = "/var/run/multus/"
+	defaultMultusRunDir           = "/run/multus/"
 )
 
 // LoadDelegateNetConfList reads DelegateNetConf from bytes

pkg/types/types.go

@@ -190,6 +190,7 @@ type ShimNetConf struct {
 
 // ControllerNetConf for the controller cni configuration
 type ControllerNetConf struct {
+	ChrootDir   string `json:"chrootDir,omitempty"`
 	ConfDir     string `json:"confDir"`
 	CNIDir      string `json:"cniDir"`
 	BinDir      string `json:"binDir"`