github/multus-cni
1
0
Fork 1
mirror of https://github.com/k8snetworkplumbingwg/multus-cni.git synced 2025-06-24 23:01:37 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #828 from s1061123/dev/chroot

Browse Source 
Add chroot option in multus-daemon
This commit is contained in:
Doug Smith 2022-04-13 15:38:56 -04:00 committed by GitHub
commit 13e4b3a1c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 269 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/multus-daemon/main.go
View File

@ -49,7 +49,7 @@ const (
defaultMultusMasterCNIFile = ""
defaultMultusNamespaceIsolation = false
defaultMultusReadinessIndicatorFile = ""
defaultMultusRunDir = "/host/var/run/multus-cni/"
defaultMultusRunDir = "/host/run/multus-cni/"
)
const (
@ -228,7 +228,7 @@ func startMultusDaemon(configFilePath string) error {
return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
}
server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir, config)
server, err := srv.NewCNIServer(daemonConfig, config)
if err != nil {
return fmt.Errorf("failed to create the server: %v", err)
}

36
deployments/multus-daemonset-thick.yml
View File

@ -102,13 +102,14 @@ metadata:
data:
daemon-config.json: |
{
"chrootDir": "/hostroot",
"confDir": "/host/etc/cni/net.d",
"logToStderr": true,
"logLevel": "debug",
"logFile": "/tmp/multus.log",
"binDir": "/host/opt/cni/bin",
"cniDir": "/host/var/lib/cni/multus",
"socketDir": "/host/var/run/multus/"
"binDir": "/opt/cni/bin",
"cniDir": "/var/lib/cni/multus",
"socketDir": "/host/run/multus/"
}
---
apiVersion: apps/v1
@ -163,16 +164,19 @@ spec:
volumeMounts:
- name: cni
mountPath: /host/etc/cni/net.d
- name: cnibin
mountPath: /host/opt/cni/bin
- name: host-var-run
mountPath: /host/var/run
- name: host-var-run-netns
mountPath: /var/run/netns
- name: host-run
mountPath: /host/run
- name: host-var-lib-cni-multus
mountPath: /var/lib/cni/multus
- name: host-run-netns
mountPath: /run/netns
mountPropagation: HostToContainer
- name: multus-daemon-config
mountPath: /etc/cni/net.d/multus.d
readOnly: true
- name: hostroot
mountPath: /hostroot
mountPropagation: HostToContainer
initContainers:
- name: install-multus-binary
image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
@ -198,15 +202,21 @@ spec:
- name: cnibin
hostPath:
path: /opt/cni/bin
- name: hostroot
hostPath:
path: /
- name: multus-daemon-config
configMap:
name: multus-daemon-config
items:
- key: daemon-config.json
path: daemon-config.json
- name: host-var-run
- name: host-run
hostPath:
path: /var/run
- name: host-var-run-netns
path: /run
- name: host-var-lib-cni-multus
hostPath:
path: /var/run/netns/
path: /var/lib/cni/multus
- name: host-run-netns
hostPath:
path: /run/netns/

2
docs/how-to-use.md
View File

@ -39,7 +39,7 @@ cat >/etc/cni/net.d/00-multus.conf <<EOF
{
"name": "multus-cni-network",
"type": "multus",
"readinessindicatorfile": "/var/run/flannel/subnet.env",
"readinessindicatorfile": "/run/flannel/subnet.env",
"delegates": [
{
"NOTE1": "This is example, wrote your CNI config in delegates",

9
docs/thick-plugin.md
View File

@ -79,7 +79,14 @@ specifies the path to the server configuration:
The server configuration is encoded in JSON, and allows the following keys:
- `"chrootDir"`: Specify the directory which points to host root from the pod. See 'Chroot configuration' section for the details.
- `"socketDir"`: Specify the location where the unix domain socket used for
client/server communication will be located. Defaults to `"/var/run/multus-cni/"`.
client/server communication will be located. Defaults to `"/run/multus-cni"`.
In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
#### Chroot configuration
In thick plugin case, delegate CNI plugin is executed by multus-daemon from Pod, hence if the delegate CNI requires resources in container host, for example unix socket or even file, then CNI plugin is failed to execute because multus-daemon runs in Pod. Multus-daemon supports "chrootDir" option which executes delegate CNI under chroot (to container host).
This configuration is enabled in deployments/multus-daemonset-thick.yml as default.

25
e2e/templates/multus-daemonset-thick.yml.j2
View File

@ -89,8 +89,8 @@ data:
"logLevel": "debug",
"logFile": "/tmp/multus.log",
"binDir": "/host/opt/cni/bin",
"cniDir": "/host/var/lib/cni/multus",
"socketDir": "/host/var/run/multus/"
"cniDir": "/var/lib/cni/multus",
"socketDir": "/host/run/multus"
}
---
apiVersion: apps/v1
@ -148,10 +148,12 @@ spec:
mountPath: /host/etc/cni/net.d
- name: cnibin
mountPath: /host/opt/cni/bin
- name: host-var-run
mountPath: /host/var/run
- name: host-var-run-netns
mountPath: /var/run/netns
- name: host-run
mountPath: /host/run
- name: host-var-lib-cni-multus
mountPath: /var/lib/cni/multus
- name: host-run-netns
mountPath: /run/netns
mountPropagation: HostToContainer
- name: multus-daemon-config
mountPath: /etc/cni/net.d/multus.d
@ -186,9 +188,12 @@ spec:
items:
- key: daemon-config.json
path: daemon-config.json
- name: host-var-run
- name: host-run
hostPath:
path: /var/run
- name: host-var-run-netns
path: /run
- name: host-var-lib-cni-multus
hostPath:
path: /var/run/netns/
path: /var/lib/cni/multus
- name: host-run-netns
hostPath:
path: /run/netns/

2
images/entrypoint.sh
View File

@ -223,7 +223,7 @@ MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
# ------------------------------- Generate a "kube-config"
# Inspired by: https://tinyurl.com/y7r2knme
SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
SERVICE_ACCOUNT_PATH=/run/secrets/kubernetes.io/serviceaccount
KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}

6
pkg/k8sclient/k8sclient.go
View File

@ -578,16 +578,16 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
if strings.HasSuffix(netname, ".conflist") {
confList, err := libcni.ConfListFromFile(netname)
if err != nil {
return nil, resourceMap, fmt.Errorf("Error loading CNI conflist file %s: %v", netname, err)
return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
}
configBytes = confList.Bytes
} else {
conf, err := libcni.ConfFromFile(netname)
if err != nil {
return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: %v", netname, err)
return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
}
if conf.Network.Type == "" {
return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
}
configBytes = conf.Bytes
}

24
pkg/netutils/netutils_test.go
View File

@ -429,7 +429,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -503,7 +503,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -577,7 +577,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -649,7 +649,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -973,7 +973,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1021,7 +1021,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1087,7 +1087,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1135,7 +1135,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1205,7 +1205,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1251,7 +1251,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1315,7 +1315,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [
@ -1361,7 +1361,7 @@ var _ = Describe("netutil cnicache function testing", func() {
{
"mac": "0a:c2:e6:3d:45:17",
"name": "net1",
"sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
"sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
}
],
"ips": [

182
pkg/server/exec_chroot.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright (c) 2021 Multus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
"sync"
"syscall"
"time"
"github.com/containernetworking/cni/pkg/invoke"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/version"
)
// ChrootExec implements invoke.Exec to execute CNI with chroot
type ChrootExec struct {
Stderr io.Writer
chrootDir string
workingDir string // working directory in the outer root
outerRoot *os.File // outer root directory
version.PluginDecoder
mu sync.Mutex
}
var _ invoke.Exec = &ChrootExec{}
func (e *ChrootExec) chroot() error {
var err error
e.workingDir, err = os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
return fmt.Errorf("getwd before chroot failed: %v", err)
}
e.outerRoot, err = os.Open("/")
if err != nil {
fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
return fmt.Errorf("getwd before chroot failed: %v", err)
}
if err := syscall.Chroot(e.chrootDir); err != nil {
fmt.Fprintf(os.Stderr, "chroot to %s failed: %v\n", e.chrootDir, err)
return fmt.Errorf("chroot to %s failed: %v", e.chrootDir, err)
}
if err := os.Chdir("/"); err != nil {
fmt.Fprintf(os.Stderr, "chdir to \"/\" failed: %v\n", err)
return fmt.Errorf("chdir to \"/\" failed: %v", err)
}
return nil
}
func (e *ChrootExec) escape() error {
if e.outerRoot == nil || e.workingDir == "" {
return nil
}
// change directory to outer root and close it
if err := syscall.Fchdir(int(e.outerRoot.Fd())); err != nil {
fmt.Fprintf(os.Stderr, "changing directory to outer root failed: %v\n", err)
return fmt.Errorf("changing directory to outer root failed: %v", err)
}
if err := e.outerRoot.Close(); err != nil {
fmt.Fprintf(os.Stderr, "closing outer root failed: %v\n", err)
return fmt.Errorf("closing outer root failed: %v", err)
}
// chroot to current directory aka "." being the outer root
if err := syscall.Chroot("."); err != nil {
fmt.Fprintf(os.Stderr, "chroot to current directory failed: %v\n", err)
return fmt.Errorf("chroot to current directory failed: %v", err)
}
if err := os.Chdir(e.workingDir); err != nil {
fmt.Fprintf(os.Stderr, "chdir to working directory failed: %v\n", err)
return fmt.Errorf("chdir to working directory failed: %v", err)
}
e.outerRoot = nil
e.workingDir = ""
return nil
}
// ExecPlugin executes CNI plugin with given environment/stdin data.
func (e *ChrootExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
// lock and do chroot to execute plugin with host root
e.mu.Lock()
defer e.mu.Unlock()
err := e.chroot()
defer e.escape()
if err != nil {
fmt.Fprintf(os.Stderr, "ExecPlugin failed at chroot: %v\n", err)
return nil, fmt.Errorf("ExecPlugin failed at chroot: %v", err)
}
stdout := &bytes.Buffer{}
stderr := &bytes.Buffer{}
c := exec.CommandContext(ctx, pluginPath)
c.Env = environ
c.Stdin = bytes.NewBuffer(stdinData)
c.Stdout = stdout
c.Stderr = stderr
// Retry the command on "text file busy" errors
for i := 0; i <= 5; i++ {
err = c.Run()
// Command succeeded
if err == nil {
break
}
// If the plugin is currently about to be written, then we wait a
// second and try it again
if strings.Contains(err.Error(), "text file busy") {
time.Sleep(time.Second)
continue
}
// All other errors except than the busy text file
return nil, e.pluginErr(err, stdout.Bytes(), stderr.Bytes())
}
// Copy stderr to caller's buffer in case plugin printed to both
// stdout and stderr for some reason. Ignore failures as stderr is
// only informational.
if e.Stderr != nil && stderr.Len() > 0 {
_, _ = stderr.WriteTo(e.Stderr)
}
return stdout.Bytes(), nil
}
func (e *ChrootExec) pluginErr(err error, stdout, stderr []byte) error {
emsg := types.Error{}
if len(stdout) == 0 {
if len(stderr) == 0 {
emsg.Msg = fmt.Sprintf("netplugin failed with no error message: %v", err)
} else {
emsg.Msg = fmt.Sprintf("netplugin failed: %q", string(stderr))
}
} else if perr := json.Unmarshal(stdout, &emsg); perr != nil {
emsg.Msg = fmt.Sprintf("netplugin failed but error parsing its diagnostic message %q: %v", string(stdout), perr)
}
return &emsg
}
// FindInPath try to find CNI plugin based on given path
func (e *ChrootExec) FindInPath(plugin string, paths []string) (string, error) {
e.mu.Lock()
defer e.mu.Unlock()
err := e.chroot()
defer e.escape()
if err != nil {
fmt.Fprintf(os.Stderr, "FindInPath failed at chroot: %v\n", err)
return "", fmt.Errorf("FindInPath failed at chroot: %v", err)
}
return invoke.FindInPath(plugin, paths)
}

13
pkg/server/server.go
View File

@ -77,13 +77,22 @@ func GetListener(socketPath string) (net.Listener, error) {
}
// NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
func NewCNIServer(rundir string, serverConfig []byte) (*Server, error) {
func NewCNIServer(daemonConfig *types.ControllerNetConf, serverConfig []byte) (*Server, error) {
kubeClient, err := k8s.InClusterK8sClient()
if err != nil {
return nil, fmt.Errorf("error getting k8s client: %v", err)
}
return newCNIServer(rundir, kubeClient, nil, serverConfig)
exec := invoke.Exec(nil)
if daemonConfig.ChrootDir != "" {
exec = &ChrootExec{
Stderr: os.Stderr,
chrootDir: daemonConfig.ChrootDir,
}
logging.Verbosef("server configured with chroot: %s", daemonConfig.ChrootDir)
}
return newCNIServer(daemonConfig.MultusSocketDir, kubeClient, exec, serverConfig)
}
func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {

17
pkg/server/shim.go
View File

@ -18,13 +18,14 @@ import (
)
const (
defaultMultusRunDir = "/var/run/multus/"
defaultMultusRunDir = "/run/multus/"
)
// CmdAdd implements the CNI spec ADD command handler
func CmdAdd(args *skel.CmdArgs) error {
response, cniVersion, err := postRequest(args)
if err != nil {
logging.Errorf("CmdAdd (shim): %v", err)
return err
}
@ -34,24 +35,24 @@ func CmdAdd(args *skel.CmdArgs) error {
// CmdCheck implements the CNI spec CHECK command handler
func CmdCheck(args *skel.CmdArgs) error {
response, cniVersion, err := postRequest(args)
_, _, err := postRequest(args)
if err != nil {
logging.Errorf("CmdCheck (shim): %v", err)
return err
}
logging.Verbosef("CmdCheck (shim): %v", *response.Result)
return cnitypes.PrintResult(response.Result, cniVersion)
return err
}
// CmdDel implements the CNI spec DEL command handler
func CmdDel(args *skel.CmdArgs) error {
response, cniVersion, err := postRequest(args)
_, _, err := postRequest(args)
if err != nil {
return err
logging.Errorf("CmdDel (shim): %v", err)
return nil
}
logging.Verbosef("CmdDel (shim): %v", *response.Result)
return cnitypes.PrintResult(response.Result, cniVersion)
return nil
}
func postRequest(args *skel.CmdArgs) (*Response, string, error) {

2
pkg/types/conf.go
View File

@ -44,7 +44,7 @@ const (
const (
// DefaultMultusDaemonConfigFile is the default path of the config file
DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json"
defaultMultusRunDir = "/var/run/multus/"
defaultMultusRunDir = "/run/multus/"
)
// LoadDelegateNetConfList reads DelegateNetConf from bytes

1
pkg/types/types.go
View File

@ -190,6 +190,7 @@ type ShimNetConf struct {
// ControllerNetConf for the controller cni configuration
type ControllerNetConf struct {
ChrootDir string `json:"chrootDir,omitempty"`
ConfDir string `json:"confDir"`
CNIDir string `json:"cniDir"`
BinDir string `json:"binDir"`