github/multus-cni
1
0
Fork 1
mirror of https://github.com/k8snetworkplumbingwg/multus-cni.git synced 2025-08-02 00:19:10 +00:00
Code Issues Projects Releases Wiki Activity

Refine multus-daemon config

Browse Source
This commit is contained in:
Tomofumi Hayashi 2022-03-23 02:36:34 +09:00
parent d1046fa1c9
commit 4180f88442
26 changed files with 775 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/kind-e2e.yml vendored
View File

@ -8,13 +8,13 @@ jobs:
include:
- docker-file: images/Dockerfile.thick
cni-version: "0.3.1"
multus-manifest: multus-thick-daemonset.yml
multus-manifest: multus-daemonset-thick.yml
- docker-file: images/Dockerfile
cni-version: "0.3.1"
multus-manifest: multus-daemonset.yml
- docker-file: images/Dockerfile.thick
cni-version: "0.4.0"
multus-manifest: multus-thick-daemonset.yml
multus-manifest: multus-daemonset-thick.yml
- docker-file: images/Dockerfile
cni-version: "0.4.0"
multus-manifest: multus-daemonset.yml

147
cmd/generate-kubeconfig/kubeconfig.go
View File

@ -1,147 +0,0 @@
// Copyright (c) 2021 Multus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"encoding/base64"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
)
const userRWPermission = 0600
const (
cniConfigDirVarName = "cni-config-dir"
k8sCAFilePathVarName = "kube-ca-file"
k8sServiceHostVarName = "k8s-service-host"
k8sServicePortVarName = "k8s-service-port"
serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
skipTLSVerifyVarName = "skip-tls-verify"
)
const (
defaultCniConfigDir = "/host/etc/cni/net.d"
defaultK8sCAFilePath = ""
defaultK8sServiceHost = ""
defaultK8sServicePort = 0
defaultSkipTLSValue = false
)
func main() {
k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
flag.Parse()
if *k8sServiceHost == defaultK8sServiceHost {
logInvalidArg("must provide the k8s service cluster port")
}
if *k8sServicePort == defaultK8sServicePort {
logInvalidArg("must provide the k8s service cluster port")
}
if *kubeCAFilePath == defaultK8sServiceHost {
*kubeCAFilePath = serviceAccountPath + "/ca.crt"
}
tlsCfg := "insecure-skip-tls-verify: true"
if !*skipTLSVerify {
kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
if err != nil {
logError("failed grabbing CA file: %w", err)
}
tlsCfg = "certificate-authority-data: " + kubeCAFileContents
}
multusConfigDir := *cniConfigDir + "/multus.d/"
if err := prepareCNIConfigDir(multusConfigDir); err != nil {
logError("failed to create CNI config dir: %w", err)
}
kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
if err != nil {
logError("failed grabbing k8s token: %w", err)
}
if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
logError("failed generating kubeconfig: %w", err)
}
}
func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
data, err := ioutil.ReadFile(pathCAFile)
if err != nil {
return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
}
return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
}
func k8sKubeConfigToken(tokenPath string) (string, error) {
data, err := ioutil.ReadFile(tokenPath)
if err != nil {
return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
}
return string(data), nil
}
func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
kubeConfigTemplate := `
# Kubeconfig file for Multus CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: %s://[%s]:%d
%s
users:
- name: multus
user:
token: "%s"
contexts:
- name: multus-context
context:
cluster: local
user: multus
current-context: multus-context
`
kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
logInfo("Generated KubeConfig: \n%s", kubeconfig)
return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
}
func prepareCNIConfigDir(cniConfigDirPath string) error {
return os.MkdirAll(cniConfigDirPath, userRWPermission)
}
func logInvalidArg(format string, values ...interface{}) {
log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
flag.PrintDefaults()
os.Exit(1)
}
func logError(format string, values ...interface{}) {
log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
os.Exit(1)
}
func logInfo(format string, values ...interface{}) {
log.Printf("INFO: %s", fmt.Sprintf(format, values...))
}

69
cmd/multus-daemon/main.go
View File

@ -26,21 +26,19 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilwait "k8s.io/apimachinery/pkg/util/wait"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
)
const (
multusPluginName = "multus"
multusConfigFileName = "00-multus.conf"
multusPluginName = "multus-shim"
)
const (
defaultCniConfigDir = "/etc/cni/net.d"
defaultMultusGlobalNamespaces = ""
defaultMultusKubeconfigPath = "/etc/cni/net.d/multus.d/multus.kubeconfig"
defaultMultusLogFile = ""
defaultMultusLogLevel = ""
defaultMultusLogToStdErr = false
@ -48,27 +46,21 @@ const (
defaultMultusNamespaceIsolation = false
defaultMultusReadinessIndicatorFile = ""
defaultMultusRunDir = "/host/var/run/multus-cni/"
defaultMultusBinDir = "/host/opt/cni/bin"
defaultMultusCNIDir = "/host/var/lib/cni/multus"
)
const (
cniConfigDirVarName = "cni-config-dir"
multusAdditionalBinDirVarName = "additional-bin-dir"
multusAutoconfigDirVarName = "multus-autoconfig-dir"
multusCNIVersion = "cni-version"
multusConfigFileVarName = "multus-conf-file"
multusGlobalNamespaces = "global-namespaces"
multusLogFile = "multus-log-file"
multusLogLevel = "multus-log-level"
multusLogToStdErr = "multus-log-to-stderr"
multusKubeconfigPath = "multus-kubeconfig-file-host"
multusMasterCNIFileVarName = "multus-master-cni-file"
multusNamespaceIsolation = "namespace-isolation"
multusReadinessIndicatorFile = "readiness-indicator-file"
multusRunDir = "multus-rundir"
multusCNIDirVarName = "cniDir"
multusBinDirVarName = "binDir"
cniConfigDirVarName = "cni-config-dir"
multusAutoconfigDirVarName = "multus-autoconfig-dir"
multusCNIVersion = "cni-version"
multusConfigFileVarName = "multus-conf-file"
multusGlobalNamespaces = "global-namespaces"
multusLogFile = "multus-log-file"
multusLogLevel = "multus-log-level"
multusLogToStdErr = "multus-log-to-stderr"
multusMasterCNIFileVarName = "multus-master-cni-file"
multusNamespaceIsolation = "namespace-isolation"
multusReadinessIndicatorFile = "readiness-indicator-file"
multusRunDir = "multus-rundir"
)
func main() {
@ -85,24 +77,14 @@ func main() {
logFile := flag.String(multusLogFile, "", "Path where to multus will log. Used only with --multus-conf-file=auto.")
cniVersion := flag.String(multusCNIVersion, "", "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
forceCNIVersion := flag.Bool("force-cni-version", false, "force to use given CNI version. only for kind-e2e testing") // this is only for kind-e2e
additionalBinDir := flag.String(multusAdditionalBinDirVarName, "", "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
readinessIndicator := flag.String(multusReadinessIndicatorFile, "", "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
multusBinDir := flag.String(multusBinDirVarName, defaultMultusBinDir, "The directory where the CNI plugin binaries are available")
multusCniDir := flag.String(multusCNIDirVarName, defaultMultusCNIDir, "The directory where the multus CNI cache is located")
configFilePath := flag.String("config", types.DefaultMultusDaemonConfigFile, "Specify the path to the multus-daemon configuration")
flag.Parse()
daemonConfig, err := types.LoadDaemonNetConf(*configFilePath)
if err != nil {
logging.Panicf("failed to load the multus-daemon configuration: %v", err)
os.Exit(1)
}
if err := startMultusDaemon(daemonConfig); err != nil {
if err := startMultusDaemon(*configFilePath); err != nil {
logging.Panicf("failed start the multus thick-plugin listener: %v", err)
os.Exit(3)
}
@ -114,8 +96,6 @@ func main() {
}
var configurationOptions []config.Option
configurationOptions = append(configurationOptions, config.WithAdditionalBinaryFileDir(*multusBinDir))
configurationOptions = append(configurationOptions, config.WithCniDir(*multusCniDir))
if *namespaceIsolation {
configurationOptions = append(
@ -142,17 +122,12 @@ func main() {
configurationOptions, config.WithLogFile(*logFile))
}
if *additionalBinDir != "" {
configurationOptions = append(
configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
}
if *readinessIndicator != defaultMultusReadinessIndicatorFile {
configurationOptions = append(
configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
}
multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, configurationOptions...)
if err != nil {
_ = logging.Errorf("Failed to create multus config: %v", err)
os.Exit(3)
@ -191,7 +166,7 @@ func main() {
defer func() {
stopChannel <- struct{}{}
}()
if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
if err := configManager.MonitorPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
_ = logging.Errorf("error watching file: %v", err)
}
}(make(chan struct{}), configWatcherDoneChannel)
@ -204,7 +179,13 @@ func main() {
}
}
func startMultusDaemon(daemonConfig *types.ControllerNetConf) error {
func startMultusDaemon(configFilePath string) error {
daemonConfig, config, err := types.LoadDaemonNetConf(configFilePath)
if err != nil {
logging.Panicf("failed to load the multus-daemon configuration: %v", err)
os.Exit(1)
}
if user, err := user.Current(); err != nil || user.Uid != "0" {
return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
}
@ -213,7 +194,7 @@ func startMultusDaemon(daemonConfig *types.ControllerNetConf) error {
return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
}
server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir)
server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir, config)
if err != nil {
return fmt.Errorf("failed to create the server: %v", err)
}

48
deployments/multus-daemonset-thick-plugin.yml → deployments/multus-daemonset-thick.yml
View File

@ -91,6 +91,26 @@ metadata:
name: multus
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-daemon-config
namespace: kube-system
labels:
tier: node
app: multus
data:
daemon-config.json: |
{
"confDir": "/host/etc/cni/net.d",
"logToStderr": true,
"logLevel": "debug",
"logFile": "/tmp/multus.log",
"binDir": "/host/opt/cni/bin",
"cniDir": "/host/var/lib/cni/multus",
"socketDir": "/host/var/run/multus/"
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
@ -131,8 +151,6 @@ spec:
- "-multus-autoconfig-dir=/host/etc/cni/net.d"
- "-multus-log-to-stderr=true"
- "-multus-log-level=verbose"
- "-binDir=/host/opt/cni/bin"
- "-cniDir=/host/var/lib/cni/multus"
resources:
requests:
cpu: "100m"
@ -152,6 +170,9 @@ spec:
- name: host-var-run-netns
mountPath: /var/run/netns
mountPropagation: HostToContainer
- name: multus-daemon-config
mountPath: /etc/cni/net.d/multus.d
readOnly: true
initContainers:
- name: install-multus-binary
image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
@ -169,23 +190,6 @@ spec:
- name: cnibin
mountPath: /host/opt/cni/bin
mountPropagation: Bidirectional
- name: generate-kubeconfig
image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
command:
- "/usr/src/multus-cni/bin/generate-kubeconfig"
args:
- "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
- "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
resources:
requests:
cpu: "10m"
memory: "15Mi"
securityContext:
privileged: true
volumeMounts:
- name: cni
mountPath: /host/etc/cni/net.d
mountPropagation: Bidirectional
terminationGracePeriodSeconds: 10
volumes:
- name: cni
@ -194,6 +198,12 @@ spec:
- name: cnibin
hostPath:
path: /opt/cni/bin
- name: multus-daemon-config
configMap:
name: multus-daemon-config
items:
- key: daemon-config.json
path: daemon-config.json
- name: host-var-run
hostPath:
path: /var/run

5
docs/configuration.md
View File

@ -50,8 +50,8 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
* `delegates` ([]map,required): number of delegate details in the Multus
@ -63,6 +63,7 @@ Multus will find network for clusterNetwork/defaultNetworks as following sequenc
1. CRD object for given network name, in 'kube-system' namespace
1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
1. Directory for CNI json config file. Multus will find alphabetically first file for the network
1. File path for CNI json confile file.
1. Multus failed to find network. Multus raise error message
## Miscellaneous config

26
docs/thick-plugin.md
View File

@ -32,6 +32,15 @@ described above:
## How to use it
### Configure Deployment
If your delegate CNI plugin requires some files which is in container host, please update
update `deployments/multus-daemonset-thick.yml` to add directory into multus-daemon pod.
For example, flannel requires `/run/flannel/subnet.env`, so you need to mount this directory
into the multus-daemon pod.
Required directory/files are different for each CNI plugin, so please refer your CNI plugin.
### Deployment
There is a dedicated multus daemonset specification for users wanting to use
@ -39,7 +48,7 @@ this thick plugin variant. This reference deployment spec of multus can be
deployed by following these commands:
```bash
kubectl apply -f deployments/multus-daemonset-thick-plugin.yml
kubectl apply -f deployments/multus-daemonset-thick.yml
```
### Command line parameters
@ -48,10 +57,14 @@ Multus thick plugin variant accepts the same
[entrypoint arguments](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/how-to-use.md#entrypoint-script-parameters)
its thin counterpart allows - with the following exceptions:
- `skip-multus-binary-copy`
- `restart-crio`
- `additional-bin-dir`
- `binDir`
- `cleanup-config-on-exit`
- `cniDir`
- `multus-kubeconfig-file-host`
- `rename-conf-file`
- `restart-crio`
- `skip-multus-binary-copy`
It is important to refer that these are command line parameters to the golang
binary; as such, they should be passed using a single dash ("-") e.g.
@ -66,12 +79,7 @@ specifies the path to the server configuration:
The server configuration is encoded in JSON, and allows the following keys:
- `"confDir"`: specifies the path to the CNI configuration directory.
- `"cniDir"`: specifies the path to the multus CNI cache.
- `"binDir"`: specifies the path to the CNI binary executables.
- `"logFile"`: specifies where the daemon log file will be persisted.
- `"logLevel"`: indicates the logging level of the multus daemon.
- `"logToStderr"`: Whether or not to also log to stderr. Default to `true`.
- `"socketDir"`: Specify the location where the unix domain socket used for
client/server communication will be located. Defaults to `"/var/run/multus-cni/"`.
In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)

1
e2e/README.md
View File

@ -7,6 +7,7 @@
$ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
$ cd multus-cni/e2e
$ ./get_tools.sh
$ ./generate_yamls.sh
$ ./setup_cluster.sh
$ ./test-simple-macvlan1.sh
```

6
e2e/setup_cluster.sh
View File

@ -8,9 +8,9 @@ export PATH=${PATH}:./bin
OCI_BIN="${OCI_BIN:-docker}"
# define the deployment spec to use when deploying multus.
# Acceptable values are `multus-daemonset.yml`. `multus-thick-daemonset.yml`.
# Defaults to `multus-thick-daemonset.yml`.
MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-thick-daemonset.yml}"
# Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
# Defaults to `multus-daemonset-thick.yml`.
MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
kind_network='kind'
reg_name='kind-registry'

74
e2e/templates/multus-thick-daemonset.yml.j2 → e2e/templates/multus-daemonset-thick.yml.j2
View File

@ -75,55 +75,6 @@ metadata:
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-cni-config
namespace: kube-system
labels:
tier: node
app: multus
data:
# NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
# In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
# change the "args" line below from
# - "--multus-conf-file=auto"
# to:
# "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
# Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
# /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
cni-conf.json: |
{
"name": "multus-cni-network",
"type": "multus",
"capabilities": {
"portMappings": true
},
"delegates": [
{
"cniVersion": "0.3.1",
"name": "default-cni-network",
"plugins": [
{
"type": "flannel",
"name": "flannel.1",
"delegate": {
"isDefaultGateway": true,
"hairpinMode": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
],
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
}
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-daemon-config
namespace: kube-system
@ -139,7 +90,7 @@ data:
"logFile": "/tmp/multus.log",
"binDir": "/host/opt/cni/bin",
"cniDir": "/host/var/lib/cni/multus",
"socketDir": "/host/var/run/multus-cni/"
"socketDir": "/host/var/run/multus/"
}
---
apiVersion: apps/v1
@ -178,10 +129,10 @@ spec:
imagePullPolicy: Always
command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
args:
- "-multus-conf-file=auto"
- "-force-cni-version=true"
- "-cni-version={{ CNI_VERSION }}"
- "-cni-config-dir=/host/etc/cni/net.d"
- "-force-cni-version=true"
- "-multus-conf-file=auto"
- "-multus-autoconfig-dir=/host/etc/cni/net.d"
resources:
requests:
@ -211,7 +162,7 @@ spec:
command:
- "cp"
- "/usr/src/multus-cni/bin/multus-shim"
- "/host/opt/cni/bin/multus"
- "/host/opt/cni/bin/multus-shim"
resources:
requests:
cpu: "10m"
@ -222,23 +173,6 @@ spec:
- name: cnibin
mountPath: /host/opt/cni/bin
mountPropagation: Bidirectional
- name: generate-kubeconfig
image: localhost:5000/multus:e2e
command:
- "/usr/src/multus-cni/bin/generate-kubeconfig"
args:
- "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
- "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
resources:
requests:
cpu: "10m"
memory: "15Mi"
securityContext:
privileged: true
volumeMounts:
- name: cni
mountPath: /host/etc/cni/net.d
mountPropagation: Bidirectional
volumes:
- name: cni
hostPath:

3
hack/build-go.sh
View File

@ -41,7 +41,6 @@ if [ "$GO111MODULE" == "off" ]; then
export GOBIN=${PWD}/bin
export GOPATH=${PWD}/gopath
go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus
go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/generate-kubeconfig
go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus-daemon
go build -o ${PWD}/bin/multus-shim -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus-shim
else
@ -54,8 +53,6 @@ else
echo "Building plugins"
go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
echo "Building spec generators"
go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/generate-kubeconfig
echo "Building multus controller"
go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/multus-daemon
echo "Building multus shim"

4
images/entrypoint.sh
View File

@ -292,6 +292,10 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
while [ $found_master == false ]; do
if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then
MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME"
if [ ! -f "$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN" ]; then
error "Cannot find master cni file $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN"
exit 1;
fi
else
MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
fi

75
pkg/k8sclient/k8sclient.go
View File

@ -522,32 +522,40 @@ func isValidNamespaceReference(targetns string, allowednamespaces []string) bool
return false
}
// getNetDelegate loads delegate network for clusterNetwork/defaultNetworks
func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace)
// option1) search CRD object for the network
net := &types.NetworkSelectionElement{
Name: netname,
Namespace: namespace,
}
delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
if err == nil {
return delegate, resourceMap, nil
}
// option2) search CNI json config file
var configBytes []byte
configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
if err == nil {
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
if err != nil {
return nil, resourceMap, err
}
return delegate, resourceMap, nil
}
isNetnamePath := strings.Contains(netname, "/")
// option3) search directory
fInfo, err := os.Stat(netname)
if err == nil {
// if netname is not directory or file, it must be net-attach-def name or CNI config name
if ! isNetnamePath {
// option1) search CRD object for the network
net := &types.NetworkSelectionElement{
Name: netname,
Namespace: namespace,
}
delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
if err == nil {
return delegate, resourceMap, nil
}
// option2) search CNI json config file, which has <netname> as CNI name, from confDir
configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
if err == nil {
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
if err != nil {
return nil, resourceMap, err
}
return delegate, resourceMap, nil
}
} else {
fInfo, err := os.Stat(netname)
if err != nil {
return nil, resourceMap, err
}
// option3) search directory
if fInfo.IsDir() {
files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
if err != nil {
@ -565,6 +573,29 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
}
return nil, resourceMap, err
}
} else {
// option4) if file path (absolute), then load it directly
if strings.HasSuffix(netname, ".conflist") {
confList, err := libcni.ConfListFromFile(netname)
if err != nil {
return nil, resourceMap, fmt.Errorf("Error loading CNI conflist file %s: %v", netname, err)
}
configBytes = confList.Bytes
} else {
conf, err := libcni.ConfFromFile(netname)
if err != nil {
return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: %v", netname, err)
}
if conf.Network.Type == "" {
return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
}
configBytes = conf.Bytes
}
delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
if err != nil {
return nil, resourceMap, err
}
return delegate, resourceMap, nil
}
}
return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)

33
pkg/k8sclient/k8sclient_test.go
View File

@ -512,7 +512,7 @@ var _ = Describe("k8sclient operations", func() {
Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
})
It("retrieves cluster network from path", func() {
It("retrieves cluster network from directory path", func() {
fakePod := testutils.NewFakePod(fakePodName, "", "")
conf := fmt.Sprintf(`{
"name":"node-cni-network",
@ -544,6 +544,37 @@ var _ = Describe("k8sclient operations", func() {
Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
})
It("retrieves cluster network from cni config path", func() {
net1Name := filepath.Join(tmpDir, "10-net1.conf")
ioutil.WriteFile(net1Name, []byte(`{
"name": "net1",
"type": "mynet",
"cniVersion": "0.3.1"
}`), 0600)
fakePod := testutils.NewFakePod(fakePodName, "", "")
conf := fmt.Sprintf(`{
"name":"node-cni-network",
"type":"multus",
"clusterNetwork": "%s",
"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
}`, net1Name)
netConf, err := types.LoadNetConf([]byte(conf))
Expect(err).NotTo(HaveOccurred())
clientInfo := NewFakeClientInfo()
_, err = clientInfo.AddPod(fakePod)
Expect(err).NotTo(HaveOccurred())
_, err = GetK8sArgs(args)
Expect(err).NotTo(HaveOccurred())
_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
Expect(err).NotTo(HaveOccurred())
Expect(len(netConf.Delegates)).To(Equal(1))
Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
})
It("Error in case of CRD not found", func() {
fakePod := testutils.NewFakePod(fakePodName, "", "")
conf := `{

4
pkg/multus/multus.go
View File

@ -612,7 +612,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
// Even if the filename is set, file may not be present. Ignore error,
// but log and in the future may need to filter on specific errors.
if err != nil {
logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
logging.Debugf("CmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
}
}
@ -704,7 +704,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
if err != nil {
// Even if the filename is set, file may not be present. Ignore error,
// but log and in the future may need to filter on specific errors.
logging.Debugf("cmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
logging.Debugf("CmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
}
// create the network status, only in case Multus as kubeconfig

28
pkg/server/config/config_suite_test.go Normal file
View File

@ -0,0 +1,28 @@
// Copyright (c) 2021 Multus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package config
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"testing"
)
func TestConfig(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "server/config")
}

101
pkg/server/config/generator.go
View File

@ -34,19 +34,18 @@ const (
)
// Option mutates the `conf` object
type Option func(conf *MultusConf)
type Option func(conf *MultusConf) error
// MultusConf holds the multus configuration, and persists it to disk
type MultusConf struct {
BinDir string `json:"binDir,omitempty"`
Capabilities map[string]bool `json:"capabilities,omitempty"`
CNIVersion string `json:"cniVersion"`
Delegates []interface{} `json:"delegates"`
LogFile string `json:"logFile,omitempty"`
LogLevel string `json:"logLevel,omitempty"`
LogToStderr bool `json:"logToStderr,omitempty"`
Kubeconfig string `json:"kubeconfig"`
Name string `json:"name"`
ClusterNetwork string `json:"clusterNetwork,omitempty"`
NamespaceIsolation bool `json:"namespaceIsolation,omitempty"`
RawNonIsolatedNamespaces string `json:"globalNamespaces,omitempty"`
ReadinessIndicatorFile string `json:"readinessindicatorfile,omitempty"`
@ -56,14 +55,12 @@ type MultusConf struct {
// NewMultusConfig creates a basic configuration generator. It can be mutated
// via the `With...` methods.
func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
func NewMultusConfig(pluginName string, cniVersion string, configurationOptions ...Option) (*MultusConf, error) {
multusConfig := &MultusConf{
Name: MultusDefaultNetworkName,
CNIVersion: cniVersion,
Type: pluginName,
Capabilities: map[string]bool{},
Kubeconfig: kubeconfig,
Delegates: []interface{}{},
}
err := multusConfig.Mutate(configurationOptions...)
@ -74,7 +71,7 @@ func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, co
// top level cni version with the delegate cni version.
// Since version 0.4.0, CHECK was introduced, which
// causes incompatibility.
func CheckVersionCompatibility(mc *MultusConf) error {
func CheckVersionCompatibility(mc *MultusConf, delegate interface{}) error {
const versionFmt = "delegate cni version is %s while top level cni version is %s"
v040, _ := semver.Make("0.4.0")
multusCNIVersion, err := semver.Make(mc.CNIVersion)
@ -84,22 +81,20 @@ func CheckVersionCompatibility(mc *MultusConf) error {
}
if multusCNIVersion.GTE(v040) {
for _, delegate := range mc.Delegates {
delegatesMap, ok := delegate.(map[string]interface{})
if !ok {
return errors.New("couldn't get cni version of delegate")
}
delegateVersion, ok := delegatesMap["cniVersion"].(string)
if !ok {
return errors.New("couldn't get cni version of delegate")
}
v, err := semver.Make(delegateVersion)
if err != nil {
return err
}
if v.LT(v040) {
return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
}
delegatesMap, ok := delegate.(map[string]interface{})
if !ok {
return errors.New("couldn't get cni version of delegate")
}
delegateVersion, ok := delegatesMap["cniVersion"].(string)
if !ok {
return errors.New("couldn't get cni version of delegate")
}
v, err := semver.Make(delegateVersion)
if err != nil {
return err
}
if v.LT(v040) {
return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
}
}
@ -117,81 +112,100 @@ func (mc *MultusConf) Generate() (string, error) {
// configuration `Option`s
func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
for _, configOption := range configurationOptions {
configOption(mc)
err := configOption(mc)
if err != nil {
return err
}
}
return CheckVersionCompatibility(mc)
return nil
}
// WithNamespaceIsolation mutates the inner state to enable the
// NamespaceIsolation attribute
func WithNamespaceIsolation() Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.NamespaceIsolation = true
return nil
}
}
// WithGlobalNamespaces mutates the inner state to set the
// RawNonIsolatedNamespaces attribute
func WithGlobalNamespaces(globalNamespaces string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.RawNonIsolatedNamespaces = globalNamespaces
return nil
}
}
// WithLogToStdErr mutates the inner state to enable the
// WithLogToStdErr attribute
func WithLogToStdErr() Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.LogToStderr = true
return nil
}
}
// WithLogLevel mutates the inner state to set the
// LogLevel attribute
func WithLogLevel(logLevel string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.LogLevel = logLevel
return nil
}
}
// WithLogFile mutates the inner state to set the
// logFile attribute
func WithLogFile(logFile string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.LogFile = logFile
return nil
}
}
// WithReadinessFileIndicator mutates the inner state to set the
// ReadinessIndicatorFile attribute
func WithReadinessFileIndicator(path string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.ReadinessIndicatorFile = path
return nil
}
}
// WithAdditionalBinaryFileDir mutates the inner state to set the
// BinDir attribute
func WithAdditionalBinaryFileDir(directoryPath string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.BinDir = directoryPath
return nil
}
}
// WithOverriddenName mutates the inner state to set the
// Name attribute
func WithOverriddenName(networkName string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.Name = networkName
return nil
}
}
// WithCniDir mutates the inner state to set the
// multus CNI cache directory
func WithCniDir(cniDir string) Option {
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
conf.CniDir = cniDir
return nil
}
}
func withClusterNetwork(clusterNetwork string) Option {
return func(conf *MultusConf) error {
conf.ClusterNetwork = clusterNetwork
return nil
}
}
@ -203,6 +217,10 @@ func withCapabilities(cniData interface{}) Option {
if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
pluginsList = pluginsListEntry.([]interface{})
}
} else {
return func(conf *MultusConf) error {
return errors.New("couldn't get cni config from delegate")
}
}
if len(pluginsList) > 0 {
@ -215,22 +233,11 @@ func withCapabilities(cniData interface{}) Option {
enabledCapabilities = extractCapabilities(cniData)
}
return func(conf *MultusConf) {
return func(conf *MultusConf) error {
for _, capability := range enabledCapabilities {
conf.Capabilities[capability] = true
}
}
}
func withDelegates(primaryCNIConfigData map[string]interface{}, cniVersion string, forceCNIVersion bool) Option {
// override delegates CNIVersion with multus CNIVersion
if forceCNIVersion {
primaryCNIConfigData["cniVersion"] = cniVersion
}
return func(conf *MultusConf) {
conf.Delegates = []interface{}{primaryCNIConfigData}
return nil
}
}

502
pkg/server/config/generator_test.go
View File

@ -18,13 +18,17 @@ package config
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
primaryCNIName = "myCNI"
cniVersion = "0.4.0"
kubeconfig = "/a/b/c/kubeconfig.kubeconfig"
)
type testCase struct {
@ -44,244 +48,308 @@ var primaryCNIConfig = map[string]interface{}{
"logfile-maxbackups": 5,
"logfile-maxage": 5,
}
var primaryCNIFile = "/etc/cni/net.d/10-flannel.conf"
func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
func newMultusConfigWithDelegates(pluginName string, cniVersion string, primaryCNIFile string, configOptions ...Option) (*MultusConf, error) {
multusConfig, err := NewMultusConfig(pluginName, cniVersion, configOptions...)
if err != nil {
return multusConfig, err
}
return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{}), "", false))
return multusConfig, multusConfig.Mutate(withClusterNetwork(primaryCNIFile))
}
func TestBasicMultusConfig(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig)
assertError(t, err, nil)
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
var _ = Describe("Configuration Generator", func() {
var tmpDir string
var err error
func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithNamespaceIsolation())
assertError(t, err, nil)
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
BeforeEach(func() {
tmpDir, err = ioutil.TempDir("", "multus_tmp")
Expect(err).NotTo(HaveOccurred())
})
func TestMultusConfigWithReadinessIndicator(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithReadinessFileIndicator("/a/b/u/it-lives"))
assertError(t, err, nil)
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
AfterEach(func() {
err := os.RemoveAll(tmpDir)
Expect(err).NotTo(HaveOccurred())
})
func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithLogLevel("notice"),
WithLogToStdErr(),
WithLogFile("/u/y/w/log.1"))
assertError(t, err, nil)
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("basic multus config", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithGlobalNamespace(t *testing.T) {
const globalNamespace = "come-along-ns"
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithGlobalNamespaces(globalNamespace))
assertError(t, err, nil)
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with namespaceisolation", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithNamespaceIsolation())
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"namespaceIsolation":true,
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
const anotherCNIBinDir = "a-dir-somewhere"
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithAdditionalBinaryFileDir(anotherCNIBinDir))
assertError(t, err, nil)
expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with readinessindicator", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithReadinessFileIndicator("/a/b/u/it-lives"))
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"readinessindicatorfile":"/a/b/u/it-lives",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithCapabilities(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true}}`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with logging configuration", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithLogLevel("notice"),
WithLogToStdErr(),
WithLogFile("/u/y/w/log.1"))
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"logFile":"/u/y/w/log.1",
"logLevel":"notice",
"logToStderr":true,
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with global namespace", func() {
const globalNamespace = "come-along-ns"
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithGlobalNamespaces(globalNamespace))
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"globalNamespaces":"come-along-ns",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with additional binDir", func() {
const anotherCNIBinDir = "a-dir-somewhere"
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithAdditionalBinaryFileDir(anotherCNIBinDir))
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"binDir":"a-dir-somewhere",
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with capabilities", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true}}`)),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{
"portMappings":true
},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with multiple capabilities", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{"portMappings":true,"tuning":true},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
withCapabilities(
documentHelper(`
{
"plugins": [
{
"capabilities": {
"portMappings": true
}
},
{
"capabilities": {
"tuning": false
}
}
]
}`)))
assertError(t, err, nil)
expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
It("multus config with multiple capabilities filter only enabled", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{"portMappings":true},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func assertError(t *testing.T, actual error, expected error) {
if actual != nil && expected != nil {
if actual.Error() != expected.Error() {
t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
}
}
It("multus config with multiple capabilities defined on a plugin", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{"portMappings":true,"tuning":true},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
if actual == nil && expected != nil {
t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
} else if actual != nil && expected == nil {
t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
}
}
It("multus config with multiple capabilities defined on multiple plugins", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`
{
"plugins": [
{
"capabilities": { "portMappings": true }
},
{
"capabilities": { "tuning": true }
}
]
}`)),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{"portMappings":true,"tuning":true},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
}
It("multus config with multiple capabilities defined on multiple plugins filter only enabled", func() {
multusConfig, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
withCapabilities(
documentHelper(`
{
"plugins": [
{
"capabilities": {
"portMappings": true
}
},
{
"capabilities": {
"tuning": false
}
}
]
}`),
),
)
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"capabilities":{"portMappings":true},
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"multus-cni-network",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
func TestVersionIncompatibility(t *testing.T) {
const delegateCNIVersion = "0.3.0"
It("multus config with overridden name", func() {
newNetworkName := "mega-net-2000"
multusConfig, _ := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
primaryCNIFile,
WithOverriddenName(newNetworkName))
Expect(err).NotTo(HaveOccurred())
expectedResult := fmt.Sprintf(`
{
"cniVersion":"0.4.0",
"clusterNetwork":"%s",
"name":"mega-net-2000",
"type":"myCNI"
}`, primaryCNIFile)
Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
})
primaryCNIConfigOld := primaryCNIConfig
tmpVer := primaryCNIConfig["cniVersion"]
primaryCNIConfig["cniVersion"] = delegateCNIVersion
_, err := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfigOld)
primaryCNIConfig["cniVersion"] = tmpVer
assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
}
func TestMultusConfigWithOverriddenName(t *testing.T) {
newNetworkName := "mega-net-2000"
multusConfig, _ := newMultusConfigWithDelegates(
primaryCNIName,
cniVersion,
kubeconfig,
primaryCNIConfig,
WithOverriddenName(newNetworkName))
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
}
func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
return &testCase{
t: t,
configGenerationFunction: configGenerationFunc,
}
}
func (tc testCase) assertResult(expectedResult string) {
multusCNIConfig, err := tc.configGenerationFunction()
if err != nil {
tc.t.Fatalf("error generating multus configuration: %v", err)
}
if multusCNIConfig != expectedResult {
tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
}
}
})
func documentHelper(pluginInfo string) interface{} {
dp, _ := documentCNIData([]byte(pluginInfo))

50
pkg/server/config/manager.go
View File

@ -41,8 +41,6 @@ type Manager struct {
multusConfigDir string
multusConfigFilePath string
primaryCNIConfigPath string
cniVersion string
forceCNIVersion bool
}
// NewManager returns a config manager object, configured to persist the
@ -65,19 +63,51 @@ func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigD
return newManager(config, multusAutoconfigDir, primaryCNIPluginName, forceCNIVersion)
}
func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
// overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case
func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
masterCNIConfigData, err := ioutil.ReadFile(cniConfigFile)
if err != nil {
return fmt.Errorf("failed to read cni config %s: %v", cniConfigFile, err)
}
var primaryCNIConfigData map[string]interface{}
if err := json.Unmarshal(masterCNIConfigData, &primaryCNIConfigData); err != nil {
return fmt.Errorf("failed to unmarshall cni config %s: %w", cniConfigFile, err)
}
primaryCNIConfigData["cniVersion"] = multusCNIVersion
configBytes, err := json.Marshal(primaryCNIConfigData)
if err != nil {
return fmt.Errorf("couldn't update cluster network config: %v", err)
}
err = ioutil.WriteFile(cniConfigFile, configBytes, 0644)
if err != nil {
return fmt.Errorf("couldn't update cluster network config: %v", err)
}
return nil
}
func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
if forceCNIVersion {
overrideCNIVersion(cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), config.CNIVersion)
}
watcher, err := newWatcher(multusConfigDir)
if err != nil {
return nil, err
}
if defaultCNIPluginName == fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) {
return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", multusConfigDir, multusConfigFileName)
}
configManager := &Manager{
configWatcher: watcher,
multusConfig: &config,
multusConfigDir: multusConfigDir,
multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
forceCNIVersion: forceCNIVersion,
}
if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
@ -92,6 +122,11 @@ func (m *Manager) loadPrimaryCNIConfigFromFile() error {
if err != nil {
return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
}
if err = CheckVersionCompatibility(m.multusConfig, primaryCNIConfigData); err != nil {
return err
}
return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
}
@ -115,7 +150,7 @@ func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface
m.cniConfigData = cniConfigData
return m.multusConfig.Mutate(
withDelegates(cniConfigData, m.multusConfig.CNIVersion, m.forceCNIVersion),
withClusterNetwork(m.primaryCNIConfigPath),
withCapabilities(cniConfigData))
}
@ -128,10 +163,10 @@ func (m Manager) GenerateConfig() (string, error) {
return m.multusConfig.Generate()
}
// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
// MonitorPluginConfiguration monitors the configuration file pointed
// to by the primaryCNIPluginName attribute, and re-generates the multus
// configuration whenever the primary CNI config is updated.
func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
func (m Manager) MonitorPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
for {
@ -143,6 +178,7 @@ func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, don
logging.Debugf("skipping un-related event %v", event)
continue
}
logging.Debugf("process event: %v", event)
if !shouldRegenerateConfig(event) {
continue

92
pkg/server/config/manager_test.go
View File

@ -19,20 +19,12 @@ import (
"fmt"
"io/ioutil"
"os"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const suiteName = "Configuration Manager"
func TestMultusConfigurationManager(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, suiteName)
}
var _ = Describe(suiteName, func() {
var _ = Describe("Configuration Manager", func() {
const (
primaryCNIPluginName = "00-mycni.conf"
primaryCNIPluginTemplate = `
@ -55,17 +47,13 @@ var _ = Describe(suiteName, func() {
multusConfigDir, err = ioutil.TempDir("", "multus-config")
Expect(err).ToNot(HaveOccurred())
Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
})
BeforeEach(func() {
defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed())
multusConf, _ := NewMultusConfig(
primaryCNIName,
cniVersion,
kubeconfig)
var err error
cniVersion)
configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
Expect(err).NotTo(HaveOccurred())
})
@ -75,52 +63,19 @@ var _ = Describe(suiteName, func() {
})
It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"multus-cni-network\",\"clusterNetwork\":\"%s\",\"type\":\"myCNI\"}", defaultCniConfig)
config, err := configManager.GenerateConfig()
Expect(err).NotTo(HaveOccurred())
Expect(config).To(Equal(expectedResult))
})
Context("Updates to the delegate CNI configuration", func() {
var (
doneChannel chan struct{}
stopChannel chan struct{}
)
BeforeEach(func() {
doneChannel = make(chan struct{})
stopChannel = make(chan struct{})
go func() {
Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
}()
})
AfterEach(func() {
go func() { stopChannel <- struct{}{} }()
Eventually(<-doneChannel).Should(Equal(struct{}{}))
close(doneChannel)
close(stopChannel)
})
It("Trigger the re-generation of the Multus CNI configuration", func() {
newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), UserRWPermission)).To(Succeed())
multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
Eventually(func() (string, error) {
multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
return string(multusCniData), err
}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
})
})
When("the user requests the name of the multus configuration to be overridden", func() {
BeforeEach(func() {
Expect(configManager.OverrideNetworkName()).To(Succeed())
})
It("Overrides the name of the multus configuration when requested", func() {
expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"mycni-name\",\"clusterNetwork\":\"%s\",\"type\":\"myCNI\"}", defaultCniConfig)
config, err := configManager.GenerateConfig()
Expect(err).NotTo(HaveOccurred())
Expect(config).To(Equal(expectedResult))
@ -128,6 +83,41 @@ var _ = Describe(suiteName, func() {
})
})
func multusConfigFromDelegate(delegateConfig string) string {
return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
var _ = Describe("Configuration Manager with mismatched cniVersion", func() {
const (
primaryCNIPluginName = "00-mycni.conf"
primaryCNIPluginTemplate = `
{
"cniVersion": "0.3.1",
"name": "mycni-name",
"type": "mycni",
"ipam": {},
"dns": {}
}
`
)
var multusConfigDir string
var defaultCniConfig string
It("test cni version incompatibility", func() {
var err error
multusConfigDir, err = ioutil.TempDir("", "multus-config")
Expect(err).ToNot(HaveOccurred())
Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed())
multusConf, _ := NewMultusConfig(
primaryCNIName,
cniVersion)
_, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
Expect(err).To(MatchError("failed to load the primary CNI configuration as a multus delegate with error 'delegate cni version is 0.3.1 while top level cni version is 0.4.0'"))
})
AfterEach(func() {
Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
})
})

45
pkg/server/server.go
View File

@ -16,6 +16,7 @@
package server
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
@ -30,10 +31,10 @@ import (
cni100 "github.com/containernetworking/cni/pkg/types/100"
"github.com/gorilla/mux"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config"
k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
)
@ -76,25 +77,33 @@ func GetListener(socketPath string) (net.Listener, error) {
}
// NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
func NewCNIServer(rundir string) (*Server, error) {
func NewCNIServer(rundir string, serverConfig []byte) (*Server, error) {
kubeClient, err := k8s.InClusterK8sClient()
if err != nil {
return nil, fmt.Errorf("error getting k8s client: %v", err)
}
return newCNIServer(rundir, kubeClient, nil)
return newCNIServer(rundir, kubeClient, nil, serverConfig)
}
func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec) (*Server, error) {
func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {
// preprocess server config to be used to override multus CNI config
// see extractCniData() for the detail
if servConfig != nil {
servConfig = bytes.Replace(servConfig, []byte("{"), []byte(","), 1)
}
router := mux.NewRouter()
s := &Server{
Server: http.Server{
Handler: router,
},
rundir: rundir,
requestFunc: HandleCNIRequest,
kubeclient: kubeClient,
exec: exec,
rundir: rundir,
requestFunc: HandleCNIRequest,
kubeclient: kubeClient,
exec: exec,
serverConfig: servConfig,
}
router.NotFoundHandler = http.HandlerFunc(http.NotFound)
@ -124,7 +133,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
if err := json.Unmarshal(b, &cr); err != nil {
return nil, err
}
cmdType, cniCmdArgs, err := extractCniData(&cr)
cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
if err != nil {
return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
}
@ -142,7 +151,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
return result, nil
}
func extractCniData(cniRequest *Request) (string, *skel.CmdArgs, error) {
func extractCniData(cniRequest *Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
cmd, ok := cniRequest.Env["CNI_COMMAND"]
if !ok {
return "", nil, fmt.Errorf("unexpected or missing CNI_COMMAND")
@ -168,7 +177,20 @@ func extractCniData(cniRequest *Request) (string, *skel.CmdArgs, error) {
return "", nil, fmt.Errorf("missing CNI_ARGS")
}
cniCmdArgs.Args = cniArgs
cniCmdArgs.StdinData = cniRequest.Config
if overrideConf != nil {
// trim the close bracket from multus CNI config and put the server config
// to override CNI config with server config.
// note: if there are two or more value in same key, then the
// latest one is used at golang json implementation
idx := bytes.LastIndex(cniRequest.Config, []byte("}"))
if idx == -1 {
return "", nil, fmt.Errorf("invalid CNI config")
}
cniCmdArgs.StdinData = append(cniRequest.Config[:idx], overrideConf...)
} else {
cniCmdArgs.StdinData = cniRequest.Config
}
return cmd, cniCmdArgs, nil
}
@ -277,6 +299,7 @@ func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, k
}
func serializeResult(result cnitypes.Result) ([]byte, error) {
// cni result is converted to latest here and decoded to specific cni version at multus-shim
realResult, err := cni100.NewResultFromResult(result)
if err != nil {
return nil, fmt.Errorf("failed to generate the CNI result: %w", err)

26
pkg/server/shim.go
View File

@ -18,64 +18,64 @@ import (
)
const (
defaultMultusRunDir = "/var/run/multus-cni/"
defaultMultusRunDir = "/var/run/multus/"
)
// CmdAdd implements the CNI spec ADD command handler
func CmdAdd(args *skel.CmdArgs) error {
response, err := postRequest(args)
response, cniVersion, err := postRequest(args)
if err != nil {
return err
}
logging.Verbosef("CmdAdd (shim): %v", *response.Result)
return cnitypes.PrintResult(response.Result, response.Result.CNIVersion)
return cnitypes.PrintResult(response.Result, cniVersion)
}
// CmdCheck implements the CNI spec CHECK command handler
func CmdCheck(args *skel.CmdArgs) error {
response, err := postRequest(args)
response, cniVersion, err := postRequest(args)
if err != nil {
return err
}
logging.Verbosef("CmdCheck (shim): %v", *response.Result)
return cnitypes.PrintResult(response.Result, response.Result.CNIVersion)
return cnitypes.PrintResult(response.Result, cniVersion)
}
// CmdDel implements the CNI spec DEL command handler
func CmdDel(args *skel.CmdArgs) error {
response, err := postRequest(args)
response, cniVersion, err := postRequest(args)
if err != nil {
return err
}
logging.Verbosef("CmdDel (shim): %v", *response.Result)
return cnitypes.PrintResult(response.Result, response.Result.CNIVersion)
return cnitypes.PrintResult(response.Result, cniVersion)
}
func postRequest(args *skel.CmdArgs) (*Response, error) {
func postRequest(args *skel.CmdArgs) (*Response, string, error) {
multusShimConfig, err := shimConfig(args.StdinData)
if err != nil {
return nil, fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err)
return nil, "", fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err)
}
cniRequest, err := newCNIRequest(args)
if err != nil {
return nil, err
return nil, multusShimConfig.CNIVersion, err
}
body, err := DoCNI("http://dummy/", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
if err != nil {
return nil, err
return nil, multusShimConfig.CNIVersion, err
}
response := &Response{}
if err = json.Unmarshal(body, response); err != nil {
err = fmt.Errorf("failed to unmarshal response '%s': %v", string(body), err)
return nil, err
return nil, multusShimConfig.CNIVersion, err
}
return response, nil
return response, multusShimConfig.CNIVersion, nil
}
// Create and fill a Request with this Plugin's environment and stdin which

2
pkg/server/socket.go
View File

@ -22,7 +22,7 @@ import (
)
const (
serverSocketName = "multus-cni.sock"
serverSocketName = "multus.sock"
fullReadWriteExecutePermissions = 0777
thickPluginSocketRunDirPermissions = 0700
)

61
pkg/server/thick_cni_test.go
View File

@ -114,7 +114,61 @@ var _ = Describe(suiteName, func() {
K8sClient = fakeK8sClient()
Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
cniServer, err = startCNIServer(thickPluginRunDir, K8sClient)
cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, nil)
Expect(err).NotTo(HaveOccurred())
netns, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
// the namespace and podUID parameters below are hard-coded in the generation function
Expect(prepareCNIEnv(netns.Path(), "test", podName, "testUID")).To(Succeed())
Expect(createFakePod(K8sClient, podName)).To(Succeed())
})
AfterEach(func() {
Expect(cniServer.Close()).To(Succeed())
Expect(teardownCNIEnv()).To(Succeed())
Expect(K8sClient.Client.CoreV1().Pods("test").Delete(
context.TODO(), podName, metav1.DeleteOptions{}))
})
It("ADD works successfully", func() {
Expect(CmdAdd(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed())
})
It("DEL works successfully", func() {
Expect(CmdDel(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed())
})
It("CHECK works successfully", func() {
Expect(CmdCheck(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed())
})
})
Context("CNI operations started from the shim with CNI config override with server config", func() {
const (
containerID = "123456789"
ifaceName = "eth0"
podName = "my-little-pod"
)
var (
cniServer *Server
K8sClient *k8s.ClientInfo
netns ns.NetNS
)
BeforeEach(func() {
var err error
K8sClient = fakeK8sClient()
dummyServerConfig := `{
"dummy_key1": "dummy_val1",
"dummy_key2": "dummy_val2"
}`
Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, []byte(dummyServerConfig))
Expect(err).NotTo(HaveOccurred())
netns, err = testutils.NewNS()
@ -204,10 +258,10 @@ func createFakePod(k8sClient *k8s.ClientInfo, podName string) error {
return err
}
func startCNIServer(runDir string, k8sClient *k8s.ClientInfo) (*Server, error) {
func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) {
const period = 0
cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{})
cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig)
if err != nil {
return nil, err
}
@ -228,6 +282,7 @@ func startCNIServer(runDir string, k8sClient *k8s.ClientInfo) (*Server, error) {
func referenceConfig(thickPluginSocketDir string) string {
const referenceConfigTemplate = `{
"cniVersion": "0.4.0",
"name": "node-cni-network",
"type": "multus",
"socketDir": "%s",

9
pkg/server/types.go
View File

@ -25,10 +25,11 @@ type Request struct {
// the CNI shim requests issued when a pod is added / removed.
type Server struct {
http.Server
requestFunc cniRequestFunc
rundir string
kubeclient *k8sclient.ClientInfo
exec invoke.Exec
requestFunc cniRequestFunc
rundir string
kubeclient *k8sclient.ClientInfo
exec invoke.Exec
serverConfig []byte
}
// Response represents the response (computed in the CNI server) for

47
pkg/types/conf.go
View File

@ -44,7 +44,7 @@ const (
const (
// DefaultMultusDaemonConfigFile is the default path of the config file
DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json"
defaultMultusRunDir = "/var/run/multus-cni/"
defaultMultusRunDir = "/var/run/multus/"
)
// LoadDelegateNetConfList reads DelegateNetConf from bytes
@ -213,26 +213,29 @@ func NewCNIRuntimeConf(containerID, sandboxID, podName, podNamespace, podUID, ne
// get CNI_ARGS and set it if it does not exist in rt.Args
cniArgs := os.Getenv("CNI_ARGS")
if cniArgs != "" {
logging.Debugf("ARGS: %s", cniArgs)
for _, arg := range strings.Split(cniArgs, ";") {
for _, keyval := range strings.Split(arg, "=") {
if len(keyval) != 2 {
logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %s %s %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval))
continue
}
logging.Debugf("arg: /%v/", arg)
envKey := string(keyval[0])
envVal := string(keyval[1])
isExists := false
for _, rtArg := range rt.Args {
if rtArg[0] == envKey {
isExists = true
}
}
if isExists != false {
logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg)
rt.Args = append(rt.Args, [2]string{envKey, envVal})
keyval := strings.Split(arg, "=")
logging.Debugf("arg: /%q/, keyval: /%q/", arg, keyval)
if len(keyval) != 2 {
logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %v %v %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval))
continue
}
envKey := string(keyval[0])
envVal := string(keyval[1])
isExists := false
for _, rtArg := range rt.Args {
if rtArg[0] == envKey {
isExists = true
}
}
if isExists != false {
logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg)
rt.Args = append(rt.Args, [2]string{envKey, envVal})
}
}
}
@ -357,7 +360,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
// the existing delegate list and all delegates executed in-order.
if len(netconf.RawDelegates) == 0 && netconf.ClusterNetwork == "" {
return nil, logging.Errorf("LoadNetConf: at least one delegate/defaultNetwork must be specified")
return nil, logging.Errorf("LoadNetConf: at least one delegate/clusterNetwork must be specified")
}
if netconf.CNIDir == "" {
@ -424,15 +427,15 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
}
// LoadDaemonNetConf loads the configuration for the multus daemon
func LoadDaemonNetConf(configPath string) (*ControllerNetConf, error) {
func LoadDaemonNetConf(configPath string) (*ControllerNetConf, []byte, error) {
config, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read the config file's contents: %w", err)
return nil, nil, fmt.Errorf("failed to read the config file's contents: %w", err)
}
daemonNetConf := &ControllerNetConf{}
if err := json.Unmarshal(config, daemonNetConf); err != nil {
return nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err)
return nil, nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err)
}
logging.SetLogStderr(daemonNetConf.LogToStderr)
@ -459,7 +462,7 @@ func LoadDaemonNetConf(configPath string) (*ControllerNetConf, error) {
daemonNetConf.MultusSocketDir = defaultMultusRunDir
}
return daemonNetConf, nil
return daemonNetConf, config, nil
}
// AddDelegates appends the new delegates to the delegates list

23
pkg/types/types.go
View File

@ -30,21 +30,24 @@ type NetConf struct {
// support chaining for master interface and IP decisions
// occurring prior to running ipvlan plugin
RawPrevResult *map[string]interface{} `json:"prevResult"`
PrevResult *cni100.Result `json:"-"`
PrevResult *cni100.Result `json:"-"`
ConfDir string `json:"confDir"`
CNIDir string `json:"cniDir"`
BinDir string `json:"binDir"`
// RawDelegates is private to the NetConf class; use Delegates instead
RawDelegates []map[string]interface{} `json:"delegates"`
Delegates []*DelegateNetConf `json:"-"`
Kubeconfig string `json:"kubeconfig"`
ClusterNetwork string `json:"clusterNetwork"`
DefaultNetworks []string `json:"defaultNetworks"`
LogFile string `json:"logFile"`
LogLevel string `json:"logLevel"`
LogToStderr bool `json:"logToStderr,omitempty"`
RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"`
RawDelegates []map[string]interface{} `json:"delegates"`
// These parameters are exclusive in one config file:
// - Delegates (directly add delegate CNI config into multus CNI config)
// - ClusterNetwork+DefaultNetworks (add CNI config through CRD, directory or file)
Delegates []*DelegateNetConf `json:"-"`
ClusterNetwork string `json:"clusterNetwork"`
DefaultNetworks []string `json:"defaultNetworks"`
Kubeconfig string `json:"kubeconfig"`
LogFile string `json:"logFile"`
LogLevel string `json:"logLevel"`
LogToStderr bool `json:"logToStderr,omitempty"`
RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"`
// Default network readiness options
ReadinessIndicatorFile string `json:"readinessindicatorfile"`
// Option to isolate the usage of CR's to the namespace in which a pod resides.