From 4180f88442f85ce754e9f7ac959519bdf19eb75c Mon Sep 17 00:00:00 2001 From: Tomofumi Hayashi Date: Wed, 23 Mar 2022 02:36:34 +0900 Subject: [PATCH] Refine multus-daemon config --- .github/workflows/kind-e2e.yml | 4 +- cmd/generate-kubeconfig/kubeconfig.go | 147 ----- cmd/multus-daemon/main.go | 69 +-- ...-plugin.yml => multus-daemonset-thick.yml} | 48 +- docs/configuration.md | 5 +- docs/thick-plugin.md | 26 +- e2e/README.md | 1 + e2e/setup_cluster.sh | 6 +- ...t.yml.j2 => multus-daemonset-thick.yml.j2} | 74 +-- hack/build-go.sh | 3 - images/entrypoint.sh | 4 + pkg/k8sclient/k8sclient.go | 75 ++- pkg/k8sclient/k8sclient_test.go | 33 +- pkg/multus/multus.go | 4 +- pkg/server/config/config_suite_test.go | 28 + pkg/server/config/generator.go | 101 ++-- pkg/server/config/generator_test.go | 502 ++++++++++-------- pkg/server/config/manager.go | 50 +- pkg/server/config/manager_test.go | 92 ++-- pkg/server/server.go | 45 +- pkg/server/shim.go | 26 +- pkg/server/socket.go | 2 +- pkg/server/thick_cni_test.go | 61 ++- pkg/server/types.go | 9 +- pkg/types/conf.go | 47 +- pkg/types/types.go | 23 +- 26 files changed, 775 insertions(+), 710 deletions(-) delete mode 100644 cmd/generate-kubeconfig/kubeconfig.go rename deployments/{multus-daemonset-thick-plugin.yml => multus-daemonset-thick.yml} (87%) rename e2e/templates/{multus-thick-daemonset.yml.j2 => multus-daemonset-thick.yml.j2} (67%) create mode 100644 pkg/server/config/config_suite_test.go diff --git a/.github/workflows/kind-e2e.yml b/.github/workflows/kind-e2e.yml index e3f0ca065..5319c2f41 100644 --- a/.github/workflows/kind-e2e.yml +++ b/.github/workflows/kind-e2e.yml @@ -8,13 +8,13 @@ jobs: include: - docker-file: images/Dockerfile.thick cni-version: "0.3.1" - multus-manifest: multus-thick-daemonset.yml + multus-manifest: multus-daemonset-thick.yml - docker-file: images/Dockerfile cni-version: "0.3.1" multus-manifest: multus-daemonset.yml - docker-file: images/Dockerfile.thick cni-version: "0.4.0" - multus-manifest: multus-thick-daemonset.yml + multus-manifest: multus-daemonset-thick.yml - docker-file: images/Dockerfile cni-version: "0.4.0" multus-manifest: multus-daemonset.yml diff --git a/cmd/generate-kubeconfig/kubeconfig.go b/cmd/generate-kubeconfig/kubeconfig.go deleted file mode 100644 index f1199097e..000000000 --- a/cmd/generate-kubeconfig/kubeconfig.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2021 Multus Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "encoding/base64" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "strings" -) - -const userRWPermission = 0600 - -const ( - cniConfigDirVarName = "cni-config-dir" - k8sCAFilePathVarName = "kube-ca-file" - k8sServiceHostVarName = "k8s-service-host" - k8sServicePortVarName = "k8s-service-port" - serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount" - skipTLSVerifyVarName = "skip-tls-verify" -) - -const ( - defaultCniConfigDir = "/host/etc/cni/net.d" - defaultK8sCAFilePath = "" - defaultK8sServiceHost = "" - defaultK8sServicePort = 0 - defaultSkipTLSValue = false -) - -func main() { - k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service") - k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service") - skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped") - kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path") - cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir") - flag.Parse() - - if *k8sServiceHost == defaultK8sServiceHost { - logInvalidArg("must provide the k8s service cluster port") - } - if *k8sServicePort == defaultK8sServicePort { - logInvalidArg("must provide the k8s service cluster port") - } - if *kubeCAFilePath == defaultK8sServiceHost { - *kubeCAFilePath = serviceAccountPath + "/ca.crt" - } - - tlsCfg := "insecure-skip-tls-verify: true" - if !*skipTLSVerify { - kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath) - if err != nil { - logError("failed grabbing CA file: %w", err) - } - tlsCfg = "certificate-authority-data: " + kubeCAFileContents - } - - multusConfigDir := *cniConfigDir + "/multus.d/" - if err := prepareCNIConfigDir(multusConfigDir); err != nil { - logError("failed to create CNI config dir: %w", err) - } - kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig" - serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token") - if err != nil { - logError("failed grabbing k8s token: %w", err) - } - if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil { - logError("failed generating kubeconfig: %w", err) - } -} - -func k8sCAFileContentsBase64(pathCAFile string) (string, error) { - data, err := ioutil.ReadFile(pathCAFile) - if err != nil { - return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err) - } - return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil -} - -func k8sKubeConfigToken(tokenPath string) (string, error) { - data, err := ioutil.ReadFile(tokenPath) - if err != nil { - return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err) - } - return string(data), nil -} - -func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error { - kubeConfigTemplate := ` -# Kubeconfig file for Multus CNI plugin. -apiVersion: v1 -kind: Config -clusters: -- name: local - cluster: - server: %s://[%s]:%d - %s -users: -- name: multus - user: - token: "%s" -contexts: -- name: multus-context - context: - cluster: local - user: multus -current-context: multus-context -` - kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken) - logInfo("Generated KubeConfig: \n%s", kubeconfig) - return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission) -} - -func prepareCNIConfigDir(cniConfigDirPath string) error { - return os.MkdirAll(cniConfigDirPath, userRWPermission) -} - -func logInvalidArg(format string, values ...interface{}) { - log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error()) - flag.PrintDefaults() - os.Exit(1) -} - -func logError(format string, values ...interface{}) { - log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error()) - os.Exit(1) -} - -func logInfo(format string, values ...interface{}) { - log.Printf("INFO: %s", fmt.Sprintf(format, values...)) -} diff --git a/cmd/multus-daemon/main.go b/cmd/multus-daemon/main.go index b26ab8a44..da7229114 100644 --- a/cmd/multus-daemon/main.go +++ b/cmd/multus-daemon/main.go @@ -26,21 +26,19 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilwait "k8s.io/apimachinery/pkg/util/wait" - "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging" srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types" ) const ( - multusPluginName = "multus" - multusConfigFileName = "00-multus.conf" + multusPluginName = "multus-shim" ) const ( defaultCniConfigDir = "/etc/cni/net.d" defaultMultusGlobalNamespaces = "" - defaultMultusKubeconfigPath = "/etc/cni/net.d/multus.d/multus.kubeconfig" defaultMultusLogFile = "" defaultMultusLogLevel = "" defaultMultusLogToStdErr = false @@ -48,27 +46,21 @@ const ( defaultMultusNamespaceIsolation = false defaultMultusReadinessIndicatorFile = "" defaultMultusRunDir = "/host/var/run/multus-cni/" - defaultMultusBinDir = "/host/opt/cni/bin" - defaultMultusCNIDir = "/host/var/lib/cni/multus" ) const ( - cniConfigDirVarName = "cni-config-dir" - multusAdditionalBinDirVarName = "additional-bin-dir" - multusAutoconfigDirVarName = "multus-autoconfig-dir" - multusCNIVersion = "cni-version" - multusConfigFileVarName = "multus-conf-file" - multusGlobalNamespaces = "global-namespaces" - multusLogFile = "multus-log-file" - multusLogLevel = "multus-log-level" - multusLogToStdErr = "multus-log-to-stderr" - multusKubeconfigPath = "multus-kubeconfig-file-host" - multusMasterCNIFileVarName = "multus-master-cni-file" - multusNamespaceIsolation = "namespace-isolation" - multusReadinessIndicatorFile = "readiness-indicator-file" - multusRunDir = "multus-rundir" - multusCNIDirVarName = "cniDir" - multusBinDirVarName = "binDir" + cniConfigDirVarName = "cni-config-dir" + multusAutoconfigDirVarName = "multus-autoconfig-dir" + multusCNIVersion = "cni-version" + multusConfigFileVarName = "multus-conf-file" + multusGlobalNamespaces = "global-namespaces" + multusLogFile = "multus-log-file" + multusLogLevel = "multus-log-level" + multusLogToStdErr = "multus-log-to-stderr" + multusMasterCNIFileVarName = "multus-master-cni-file" + multusNamespaceIsolation = "namespace-isolation" + multusReadinessIndicatorFile = "readiness-indicator-file" + multusRunDir = "multus-rundir" ) func main() { @@ -85,24 +77,14 @@ func main() { logFile := flag.String(multusLogFile, "", "Path where to multus will log. Used only with --multus-conf-file=auto.") cniVersion := flag.String(multusCNIVersion, "", "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.") forceCNIVersion := flag.Bool("force-cni-version", false, "force to use given CNI version. only for kind-e2e testing") // this is only for kind-e2e - additionalBinDir := flag.String(multusAdditionalBinDirVarName, "", "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.") readinessIndicator := flag.String(multusReadinessIndicatorFile, "", "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.") - multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig") overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI") - multusBinDir := flag.String(multusBinDirVarName, defaultMultusBinDir, "The directory where the CNI plugin binaries are available") - multusCniDir := flag.String(multusCNIDirVarName, defaultMultusCNIDir, "The directory where the multus CNI cache is located") configFilePath := flag.String("config", types.DefaultMultusDaemonConfigFile, "Specify the path to the multus-daemon configuration") flag.Parse() - daemonConfig, err := types.LoadDaemonNetConf(*configFilePath) - if err != nil { - logging.Panicf("failed to load the multus-daemon configuration: %v", err) - os.Exit(1) - } - - if err := startMultusDaemon(daemonConfig); err != nil { + if err := startMultusDaemon(*configFilePath); err != nil { logging.Panicf("failed start the multus thick-plugin listener: %v", err) os.Exit(3) } @@ -114,8 +96,6 @@ func main() { } var configurationOptions []config.Option - configurationOptions = append(configurationOptions, config.WithAdditionalBinaryFileDir(*multusBinDir)) - configurationOptions = append(configurationOptions, config.WithCniDir(*multusCniDir)) if *namespaceIsolation { configurationOptions = append( @@ -142,17 +122,12 @@ func main() { configurationOptions, config.WithLogFile(*logFile)) } - if *additionalBinDir != "" { - configurationOptions = append( - configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir)) - } - if *readinessIndicator != defaultMultusReadinessIndicatorFile { configurationOptions = append( configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator)) } - multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...) + multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, configurationOptions...) if err != nil { _ = logging.Errorf("Failed to create multus config: %v", err) os.Exit(3) @@ -191,7 +166,7 @@ func main() { defer func() { stopChannel <- struct{}{} }() - if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil { + if err := configManager.MonitorPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil { _ = logging.Errorf("error watching file: %v", err) } }(make(chan struct{}), configWatcherDoneChannel) @@ -204,7 +179,13 @@ func main() { } } -func startMultusDaemon(daemonConfig *types.ControllerNetConf) error { +func startMultusDaemon(configFilePath string) error { + daemonConfig, config, err := types.LoadDaemonNetConf(configFilePath) + if err != nil { + logging.Panicf("failed to load the multus-daemon configuration: %v", err) + os.Exit(1) + } + if user, err := user.Current(); err != nil || user.Uid != "0" { return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid) } @@ -213,7 +194,7 @@ func startMultusDaemon(daemonConfig *types.ControllerNetConf) error { return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err) } - server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir) + server, err := srv.NewCNIServer(daemonConfig.MultusSocketDir, config) if err != nil { return fmt.Errorf("failed to create the server: %v", err) } diff --git a/deployments/multus-daemonset-thick-plugin.yml b/deployments/multus-daemonset-thick.yml similarity index 87% rename from deployments/multus-daemonset-thick-plugin.yml rename to deployments/multus-daemonset-thick.yml index 9cd98f4d4..b0904120b 100644 --- a/deployments/multus-daemonset-thick-plugin.yml +++ b/deployments/multus-daemonset-thick.yml @@ -91,6 +91,26 @@ metadata: name: multus namespace: kube-system --- +kind: ConfigMap +apiVersion: v1 +metadata: + name: multus-daemon-config + namespace: kube-system + labels: + tier: node + app: multus +data: + daemon-config.json: | + { + "confDir": "/host/etc/cni/net.d", + "logToStderr": true, + "logLevel": "debug", + "logFile": "/tmp/multus.log", + "binDir": "/host/opt/cni/bin", + "cniDir": "/host/var/lib/cni/multus", + "socketDir": "/host/var/run/multus/" + } +--- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -131,8 +151,6 @@ spec: - "-multus-autoconfig-dir=/host/etc/cni/net.d" - "-multus-log-to-stderr=true" - "-multus-log-level=verbose" - - "-binDir=/host/opt/cni/bin" - - "-cniDir=/host/var/lib/cni/multus" resources: requests: cpu: "100m" @@ -152,6 +170,9 @@ spec: - name: host-var-run-netns mountPath: /var/run/netns mountPropagation: HostToContainer + - name: multus-daemon-config + mountPath: /etc/cni/net.d/multus.d + readOnly: true initContainers: - name: install-multus-binary image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick @@ -169,23 +190,6 @@ spec: - name: cnibin mountPath: /host/opt/cni/bin mountPropagation: Bidirectional - - name: generate-kubeconfig - image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick - command: - - "/usr/src/multus-cni/bin/generate-kubeconfig" - args: - - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)" - - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)" - resources: - requests: - cpu: "10m" - memory: "15Mi" - securityContext: - privileged: true - volumeMounts: - - name: cni - mountPath: /host/etc/cni/net.d - mountPropagation: Bidirectional terminationGracePeriodSeconds: 10 volumes: - name: cni @@ -194,6 +198,12 @@ spec: - name: cnibin hostPath: path: /opt/cni/bin + - name: multus-daemon-config + configMap: + name: multus-daemon-config + items: + - key: daemon-config.json + path: daemon-config.json - name: host-var-run hostPath: path: /var/run diff --git a/docs/configuration.md b/docs/configuration.md index 5b6a8cf68..bad9c4377 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,8 +50,8 @@ Following is the example of multus config file, in `/etc/cni/net.d/`. User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`): -* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file -* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file +* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file +* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file * `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added) * `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` * `delegates` ([]map,required): number of delegate details in the Multus @@ -63,6 +63,7 @@ Multus will find network for clusterNetwork/defaultNetworks as following sequenc 1. CRD object for given network name, in 'kube-system' namespace 1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`) 1. Directory for CNI json config file. Multus will find alphabetically first file for the network +1. File path for CNI json confile file. 1. Multus failed to find network. Multus raise error message ## Miscellaneous config diff --git a/docs/thick-plugin.md b/docs/thick-plugin.md index 5eb5df08a..5b7a3332c 100644 --- a/docs/thick-plugin.md +++ b/docs/thick-plugin.md @@ -32,6 +32,15 @@ described above: ## How to use it +### Configure Deployment + +If your delegate CNI plugin requires some files which is in container host, please update +update `deployments/multus-daemonset-thick.yml` to add directory into multus-daemon pod. +For example, flannel requires `/run/flannel/subnet.env`, so you need to mount this directory +into the multus-daemon pod. + +Required directory/files are different for each CNI plugin, so please refer your CNI plugin. + ### Deployment There is a dedicated multus daemonset specification for users wanting to use @@ -39,7 +48,7 @@ this thick plugin variant. This reference deployment spec of multus can be deployed by following these commands: ```bash -kubectl apply -f deployments/multus-daemonset-thick-plugin.yml +kubectl apply -f deployments/multus-daemonset-thick.yml ``` ### Command line parameters @@ -48,10 +57,14 @@ Multus thick plugin variant accepts the same [entrypoint arguments](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/how-to-use.md#entrypoint-script-parameters) its thin counterpart allows - with the following exceptions: -- `skip-multus-binary-copy` -- `restart-crio` +- `additional-bin-dir` +- `binDir` - `cleanup-config-on-exit` +- `cniDir` +- `multus-kubeconfig-file-host` - `rename-conf-file` +- `restart-crio` +- `skip-multus-binary-copy` It is important to refer that these are command line parameters to the golang binary; as such, they should be passed using a single dash ("-") e.g. @@ -66,12 +79,7 @@ specifies the path to the server configuration: The server configuration is encoded in JSON, and allows the following keys: -- `"confDir"`: specifies the path to the CNI configuration directory. -- `"cniDir"`: specifies the path to the multus CNI cache. -- `"binDir"`: specifies the path to the CNI binary executables. -- `"logFile"`: specifies where the daemon log file will be persisted. -- `"logLevel"`: indicates the logging level of the multus daemon. -- `"logToStderr"`: Whether or not to also log to stderr. Default to `true`. - `"socketDir"`: Specify the location where the unix domain socket used for client/server communication will be located. Defaults to `"/var/run/multus-cni/"`. +In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`) diff --git a/e2e/README.md b/e2e/README.md index 1b948d746..de6634acf 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -7,6 +7,7 @@ $ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git $ cd multus-cni/e2e $ ./get_tools.sh +$ ./generate_yamls.sh $ ./setup_cluster.sh $ ./test-simple-macvlan1.sh ``` diff --git a/e2e/setup_cluster.sh b/e2e/setup_cluster.sh index 933ba3af2..659fca362 100755 --- a/e2e/setup_cluster.sh +++ b/e2e/setup_cluster.sh @@ -8,9 +8,9 @@ export PATH=${PATH}:./bin OCI_BIN="${OCI_BIN:-docker}" # define the deployment spec to use when deploying multus. -# Acceptable values are `multus-daemonset.yml`. `multus-thick-daemonset.yml`. -# Defaults to `multus-thick-daemonset.yml`. -MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-thick-daemonset.yml}" +# Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`. +# Defaults to `multus-daemonset-thick.yml`. +MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}" kind_network='kind' reg_name='kind-registry' diff --git a/e2e/templates/multus-thick-daemonset.yml.j2 b/e2e/templates/multus-daemonset-thick.yml.j2 similarity index 67% rename from e2e/templates/multus-thick-daemonset.yml.j2 rename to e2e/templates/multus-daemonset-thick.yml.j2 index 2ab9844ce..8fb8adb3b 100644 --- a/e2e/templates/multus-thick-daemonset.yml.j2 +++ b/e2e/templates/multus-daemonset-thick.yml.j2 @@ -75,55 +75,6 @@ metadata: --- kind: ConfigMap apiVersion: v1 -metadata: - name: multus-cni-config - namespace: kube-system - labels: - tier: node - app: multus -data: - # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. - # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod - # change the "args" line below from - # - "--multus-conf-file=auto" - # to: - # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" - # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the - # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. - cni-conf.json: | - { - "name": "multus-cni-network", - "type": "multus", - "capabilities": { - "portMappings": true - }, - "delegates": [ - { - "cniVersion": "0.3.1", - "name": "default-cni-network", - "plugins": [ - { - "type": "flannel", - "name": "flannel.1", - "delegate": { - "isDefaultGateway": true, - "hairpinMode": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - ], - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" - } ---- -kind: ConfigMap -apiVersion: v1 metadata: name: multus-daemon-config namespace: kube-system @@ -139,7 +90,7 @@ data: "logFile": "/tmp/multus.log", "binDir": "/host/opt/cni/bin", "cniDir": "/host/var/lib/cni/multus", - "socketDir": "/host/var/run/multus-cni/" + "socketDir": "/host/var/run/multus/" } --- apiVersion: apps/v1 @@ -178,10 +129,10 @@ spec: imagePullPolicy: Always command: [ "/usr/src/multus-cni/bin/multus-daemon" ] args: - - "-multus-conf-file=auto" - - "-force-cni-version=true" - "-cni-version={{ CNI_VERSION }}" - "-cni-config-dir=/host/etc/cni/net.d" + - "-force-cni-version=true" + - "-multus-conf-file=auto" - "-multus-autoconfig-dir=/host/etc/cni/net.d" resources: requests: @@ -211,7 +162,7 @@ spec: command: - "cp" - "/usr/src/multus-cni/bin/multus-shim" - - "/host/opt/cni/bin/multus" + - "/host/opt/cni/bin/multus-shim" resources: requests: cpu: "10m" @@ -222,23 +173,6 @@ spec: - name: cnibin mountPath: /host/opt/cni/bin mountPropagation: Bidirectional - - name: generate-kubeconfig - image: localhost:5000/multus:e2e - command: - - "/usr/src/multus-cni/bin/generate-kubeconfig" - args: - - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)" - - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)" - resources: - requests: - cpu: "10m" - memory: "15Mi" - securityContext: - privileged: true - volumeMounts: - - name: cni - mountPath: /host/etc/cni/net.d - mountPropagation: Bidirectional volumes: - name: cni hostPath: diff --git a/hack/build-go.sh b/hack/build-go.sh index 95a24cd18..2431c7fb7 100755 --- a/hack/build-go.sh +++ b/hack/build-go.sh @@ -41,7 +41,6 @@ if [ "$GO111MODULE" == "off" ]; then export GOBIN=${PWD}/bin export GOPATH=${PWD}/gopath go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus - go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/generate-kubeconfig go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus-daemon go build -o ${PWD}/bin/multus-shim -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/multus-shim else @@ -54,8 +53,6 @@ else echo "Building plugins" go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd/multus - echo "Building spec generators" - go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/generate-kubeconfig echo "Building multus controller" go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/multus-daemon echo "Building multus shim" diff --git a/images/entrypoint.sh b/images/entrypoint.sh index 5ec16690d..f08702541 100755 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -292,6 +292,10 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then while [ $found_master == false ]; do if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME" + if [ ! -f "$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN" ]; then + error "Cannot find master cni file $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN" + exit 1; + fi else MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)" fi diff --git a/pkg/k8sclient/k8sclient.go b/pkg/k8sclient/k8sclient.go index 80e968de0..b477d540e 100644 --- a/pkg/k8sclient/k8sclient.go +++ b/pkg/k8sclient/k8sclient.go @@ -522,32 +522,40 @@ func isValidNamespaceReference(targetns string, allowednamespaces []string) bool return false } +// getNetDelegate loads delegate network for clusterNetwork/defaultNetworks func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) { logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace) - // option1) search CRD object for the network - net := &types.NetworkSelectionElement{ - Name: netname, - Namespace: namespace, - } - delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap) - if err == nil { - return delegate, resourceMap, nil - } - - // option2) search CNI json config file var configBytes []byte - configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir) - if err == nil { - delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "") - if err != nil { - return nil, resourceMap, err - } - return delegate, resourceMap, nil - } + isNetnamePath := strings.Contains(netname, "/") - // option3) search directory - fInfo, err := os.Stat(netname) - if err == nil { + // if netname is not directory or file, it must be net-attach-def name or CNI config name + if ! isNetnamePath { + // option1) search CRD object for the network + net := &types.NetworkSelectionElement{ + Name: netname, + Namespace: namespace, + } + delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap) + if err == nil { + return delegate, resourceMap, nil + } + + // option2) search CNI json config file, which has as CNI name, from confDir + configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir) + if err == nil { + delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "") + if err != nil { + return nil, resourceMap, err + } + return delegate, resourceMap, nil + } + } else { + fInfo, err := os.Stat(netname) + if err != nil { + return nil, resourceMap, err + } + + // option3) search directory if fInfo.IsDir() { files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"}) if err != nil { @@ -565,6 +573,29 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace } return nil, resourceMap, err } + } else { + // option4) if file path (absolute), then load it directly + if strings.HasSuffix(netname, ".conflist") { + confList, err := libcni.ConfListFromFile(netname) + if err != nil { + return nil, resourceMap, fmt.Errorf("Error loading CNI conflist file %s: %v", netname, err) + } + configBytes = confList.Bytes + } else { + conf, err := libcni.ConfFromFile(netname) + if err != nil { + return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: %v", netname, err) + } + if conf.Network.Type == "" { + return nil, resourceMap, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname) + } + configBytes = conf.Bytes + } + delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "") + if err != nil { + return nil, resourceMap, err + } + return delegate, resourceMap, nil } } return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname) diff --git a/pkg/k8sclient/k8sclient_test.go b/pkg/k8sclient/k8sclient_test.go index 459bd74c3..86b2522f7 100644 --- a/pkg/k8sclient/k8sclient_test.go +++ b/pkg/k8sclient/k8sclient_test.go @@ -512,7 +512,7 @@ var _ = Describe("k8sclient operations", func() { Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet")) }) - It("retrieves cluster network from path", func() { + It("retrieves cluster network from directory path", func() { fakePod := testutils.NewFakePod(fakePodName, "", "") conf := fmt.Sprintf(`{ "name":"node-cni-network", @@ -544,6 +544,37 @@ var _ = Describe("k8sclient operations", func() { Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet")) }) + It("retrieves cluster network from cni config path", func() { + net1Name := filepath.Join(tmpDir, "10-net1.conf") + ioutil.WriteFile(net1Name, []byte(`{ + "name": "net1", + "type": "mynet", + "cniVersion": "0.3.1" + }`), 0600) + + fakePod := testutils.NewFakePod(fakePodName, "", "") + conf := fmt.Sprintf(`{ + "name":"node-cni-network", + "type":"multus", + "clusterNetwork": "%s", + "kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml" + }`, net1Name) + netConf, err := types.LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + _, err = GetK8sArgs(args) + Expect(err).NotTo(HaveOccurred()) + + _, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(len(netConf.Delegates)).To(Equal(1)) + Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1")) + Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet")) + }) + It("Error in case of CRD not found", func() { fakePod := testutils.NewFakePod(fakePodName, "", "") conf := `{ diff --git a/pkg/multus/multus.go b/pkg/multus/multus.go index c29c91cf9..16ab3b922 100644 --- a/pkg/multus/multus.go +++ b/pkg/multus/multus.go @@ -612,7 +612,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c // Even if the filename is set, file may not be present. Ignore error, // but log and in the future may need to filter on specific errors. if err != nil { - logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err) + logging.Debugf("CmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err) } } @@ -704,7 +704,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c if err != nil { // Even if the filename is set, file may not be present. Ignore error, // but log and in the future may need to filter on specific errors. - logging.Debugf("cmdAdd: getDelegateDeviceInfo returned an error - err=%v", err) + logging.Debugf("CmdAdd: getDelegateDeviceInfo returned an error - err=%v", err) } // create the network status, only in case Multus as kubeconfig diff --git a/pkg/server/config/config_suite_test.go b/pkg/server/config/config_suite_test.go new file mode 100644 index 000000000..67c86a5d3 --- /dev/null +++ b/pkg/server/config/config_suite_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2021 Multus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "server/config") +} diff --git a/pkg/server/config/generator.go b/pkg/server/config/generator.go index b5ca19751..d5a746399 100644 --- a/pkg/server/config/generator.go +++ b/pkg/server/config/generator.go @@ -34,19 +34,18 @@ const ( ) // Option mutates the `conf` object -type Option func(conf *MultusConf) +type Option func(conf *MultusConf) error // MultusConf holds the multus configuration, and persists it to disk type MultusConf struct { BinDir string `json:"binDir,omitempty"` Capabilities map[string]bool `json:"capabilities,omitempty"` CNIVersion string `json:"cniVersion"` - Delegates []interface{} `json:"delegates"` LogFile string `json:"logFile,omitempty"` LogLevel string `json:"logLevel,omitempty"` LogToStderr bool `json:"logToStderr,omitempty"` - Kubeconfig string `json:"kubeconfig"` Name string `json:"name"` + ClusterNetwork string `json:"clusterNetwork,omitempty"` NamespaceIsolation bool `json:"namespaceIsolation,omitempty"` RawNonIsolatedNamespaces string `json:"globalNamespaces,omitempty"` ReadinessIndicatorFile string `json:"readinessindicatorfile,omitempty"` @@ -56,14 +55,12 @@ type MultusConf struct { // NewMultusConfig creates a basic configuration generator. It can be mutated // via the `With...` methods. -func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) { +func NewMultusConfig(pluginName string, cniVersion string, configurationOptions ...Option) (*MultusConf, error) { multusConfig := &MultusConf{ Name: MultusDefaultNetworkName, CNIVersion: cniVersion, Type: pluginName, Capabilities: map[string]bool{}, - Kubeconfig: kubeconfig, - Delegates: []interface{}{}, } err := multusConfig.Mutate(configurationOptions...) @@ -74,7 +71,7 @@ func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, co // top level cni version with the delegate cni version. // Since version 0.4.0, CHECK was introduced, which // causes incompatibility. -func CheckVersionCompatibility(mc *MultusConf) error { +func CheckVersionCompatibility(mc *MultusConf, delegate interface{}) error { const versionFmt = "delegate cni version is %s while top level cni version is %s" v040, _ := semver.Make("0.4.0") multusCNIVersion, err := semver.Make(mc.CNIVersion) @@ -84,22 +81,20 @@ func CheckVersionCompatibility(mc *MultusConf) error { } if multusCNIVersion.GTE(v040) { - for _, delegate := range mc.Delegates { - delegatesMap, ok := delegate.(map[string]interface{}) - if !ok { - return errors.New("couldn't get cni version of delegate") - } - delegateVersion, ok := delegatesMap["cniVersion"].(string) - if !ok { - return errors.New("couldn't get cni version of delegate") - } - v, err := semver.Make(delegateVersion) - if err != nil { - return err - } - if v.LT(v040) { - return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion) - } + delegatesMap, ok := delegate.(map[string]interface{}) + if !ok { + return errors.New("couldn't get cni version of delegate") + } + delegateVersion, ok := delegatesMap["cniVersion"].(string) + if !ok { + return errors.New("couldn't get cni version of delegate") + } + v, err := semver.Make(delegateVersion) + if err != nil { + return err + } + if v.LT(v040) { + return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion) } } @@ -117,81 +112,100 @@ func (mc *MultusConf) Generate() (string, error) { // configuration `Option`s func (mc *MultusConf) Mutate(configurationOptions ...Option) error { for _, configOption := range configurationOptions { - configOption(mc) + err := configOption(mc) + if err != nil { + return err + } } - return CheckVersionCompatibility(mc) + return nil } // WithNamespaceIsolation mutates the inner state to enable the // NamespaceIsolation attribute func WithNamespaceIsolation() Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.NamespaceIsolation = true + return nil } } // WithGlobalNamespaces mutates the inner state to set the // RawNonIsolatedNamespaces attribute func WithGlobalNamespaces(globalNamespaces string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.RawNonIsolatedNamespaces = globalNamespaces + return nil } } // WithLogToStdErr mutates the inner state to enable the // WithLogToStdErr attribute func WithLogToStdErr() Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.LogToStderr = true + return nil } } // WithLogLevel mutates the inner state to set the // LogLevel attribute func WithLogLevel(logLevel string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.LogLevel = logLevel + return nil } } // WithLogFile mutates the inner state to set the // logFile attribute func WithLogFile(logFile string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.LogFile = logFile + return nil } } // WithReadinessFileIndicator mutates the inner state to set the // ReadinessIndicatorFile attribute func WithReadinessFileIndicator(path string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.ReadinessIndicatorFile = path + return nil } } // WithAdditionalBinaryFileDir mutates the inner state to set the // BinDir attribute func WithAdditionalBinaryFileDir(directoryPath string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.BinDir = directoryPath + return nil } } // WithOverriddenName mutates the inner state to set the // Name attribute func WithOverriddenName(networkName string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.Name = networkName + return nil } } // WithCniDir mutates the inner state to set the // multus CNI cache directory func WithCniDir(cniDir string) Option { - return func(conf *MultusConf) { + return func(conf *MultusConf) error { conf.CniDir = cniDir + return nil + } +} + +func withClusterNetwork(clusterNetwork string) Option { + return func(conf *MultusConf) error { + conf.ClusterNetwork = clusterNetwork + return nil } } @@ -203,6 +217,10 @@ func withCapabilities(cniData interface{}) Option { if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok { pluginsList = pluginsListEntry.([]interface{}) } + } else { + return func(conf *MultusConf) error { + return errors.New("couldn't get cni config from delegate") + } } if len(pluginsList) > 0 { @@ -215,22 +233,11 @@ func withCapabilities(cniData interface{}) Option { enabledCapabilities = extractCapabilities(cniData) } - return func(conf *MultusConf) { + return func(conf *MultusConf) error { for _, capability := range enabledCapabilities { conf.Capabilities[capability] = true } - } -} - -func withDelegates(primaryCNIConfigData map[string]interface{}, cniVersion string, forceCNIVersion bool) Option { - - // override delegates CNIVersion with multus CNIVersion - if forceCNIVersion { - primaryCNIConfigData["cniVersion"] = cniVersion - } - - return func(conf *MultusConf) { - conf.Delegates = []interface{}{primaryCNIConfigData} + return nil } } diff --git a/pkg/server/config/generator_test.go b/pkg/server/config/generator_test.go index b22741ba5..2231c482f 100644 --- a/pkg/server/config/generator_test.go +++ b/pkg/server/config/generator_test.go @@ -18,13 +18,17 @@ package config import ( "encoding/json" "fmt" + "io/ioutil" + "os" "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) const ( primaryCNIName = "myCNI" cniVersion = "0.4.0" - kubeconfig = "/a/b/c/kubeconfig.kubeconfig" ) type testCase struct { @@ -44,244 +48,308 @@ var primaryCNIConfig = map[string]interface{}{ "logfile-maxbackups": 5, "logfile-maxage": 5, } +var primaryCNIFile = "/etc/cni/net.d/10-flannel.conf" -func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) { - multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...) +func newMultusConfigWithDelegates(pluginName string, cniVersion string, primaryCNIFile string, configOptions ...Option) (*MultusConf, error) { + multusConfig, err := NewMultusConfig(pluginName, cniVersion, configOptions...) if err != nil { return multusConfig, err } - return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{}), "", false)) + return multusConfig, multusConfig.Mutate(withClusterNetwork(primaryCNIFile)) } -func TestBasicMultusConfig(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig) - assertError(t, err, nil) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} +var _ = Describe("Configuration Generator", func() { + var tmpDir string + var err error -func TestMultusConfigWithNamespaceIsolation(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithNamespaceIsolation()) - assertError(t, err, nil) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + BeforeEach(func() { + tmpDir, err = ioutil.TempDir("", "multus_tmp") + Expect(err).NotTo(HaveOccurred()) + }) -func TestMultusConfigWithReadinessIndicator(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithReadinessFileIndicator("/a/b/u/it-lives")) - assertError(t, err, nil) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + AfterEach(func() { + err := os.RemoveAll(tmpDir) + Expect(err).NotTo(HaveOccurred()) + }) -func TestMultusConfigWithLoggingConfiguration(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithLogLevel("notice"), - WithLogToStdErr(), - WithLogFile("/u/y/w/log.1")) - assertError(t, err, nil) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("basic multus config", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithGlobalNamespace(t *testing.T) { - const globalNamespace = "come-along-ns" - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithGlobalNamespaces(globalNamespace)) - assertError(t, err, nil) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with namespaceisolation", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithNamespaceIsolation()) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "namespaceIsolation":true, + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithAdditionalBinDir(t *testing.T) { - const anotherCNIBinDir = "a-dir-somewhere" - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithAdditionalBinaryFileDir(anotherCNIBinDir)) - assertError(t, err, nil) - expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with readinessindicator", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithReadinessFileIndicator("/a/b/u/it-lives")) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "readinessindicatorfile":"/a/b/u/it-lives", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithCapabilities(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(`{"capabilities": {"portMappings": true}}`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with logging configuration", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithLogLevel("notice"), + WithLogToStdErr(), + WithLogFile("/u/y/w/log.1")) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "logFile":"/u/y/w/log.1", + "logLevel":"notice", + "logToStderr":true, + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithMultipleCapabilities(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with global namespace", func() { + const globalNamespace = "come-along-ns" + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithGlobalNamespaces(globalNamespace)) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "globalNamespaces":"come-along-ns", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with additional binDir", func() { + const anotherCNIBinDir = "a-dir-somewhere" + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithAdditionalBinaryFileDir(anotherCNIBinDir)) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "binDir":"a-dir-somewhere", + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with capabilities", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(`{"capabilities": {"portMappings": true}}`)), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{ + "portMappings":true + }, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with multiple capabilities", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{"portMappings":true,"tuning":true}, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) { - multusConfig, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - withCapabilities( - documentHelper(` -{ - "plugins": [ - { - "capabilities": { - "portMappings": true - } - }, - { - "capabilities": { - "tuning": false - } - } - ] -}`))) - assertError(t, err, nil) - expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} + It("multus config with multiple capabilities filter only enabled", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{"portMappings":true}, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func assertError(t *testing.T, actual error, expected error) { - if actual != nil && expected != nil { - if actual.Error() != expected.Error() { - t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error()) - } - } + It("multus config with multiple capabilities defined on a plugin", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{"portMappings":true,"tuning":true}, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) - if actual == nil && expected != nil { - t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error()) - } else if actual != nil && expected == nil { - t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error()) - } -} + It("multus config with multiple capabilities defined on multiple plugins", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(` + { + "plugins": [ + { + "capabilities": { "portMappings": true } + }, + { + "capabilities": { "tuning": true } + } + ] + }`)), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{"portMappings":true,"tuning":true}, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error { - return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion) -} + It("multus config with multiple capabilities defined on multiple plugins filter only enabled", func() { + multusConfig, err := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + withCapabilities( + documentHelper(` + { + "plugins": [ + { + "capabilities": { + "portMappings": true + } + }, + { + "capabilities": { + "tuning": false + } + } + ] + }`), + ), + ) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "capabilities":{"portMappings":true}, + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"multus-cni-network", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) -func TestVersionIncompatibility(t *testing.T) { - const delegateCNIVersion = "0.3.0" + It("multus config with overridden name", func() { + newNetworkName := "mega-net-2000" + multusConfig, _ := newMultusConfigWithDelegates( + primaryCNIName, + cniVersion, + primaryCNIFile, + WithOverriddenName(newNetworkName)) + Expect(err).NotTo(HaveOccurred()) + expectedResult := fmt.Sprintf(` + { + "cniVersion":"0.4.0", + "clusterNetwork":"%s", + "name":"mega-net-2000", + "type":"myCNI" + }`, primaryCNIFile) + Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult)) + }) - primaryCNIConfigOld := primaryCNIConfig - tmpVer := primaryCNIConfig["cniVersion"] - primaryCNIConfig["cniVersion"] = delegateCNIVersion - _, err := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfigOld) - primaryCNIConfig["cniVersion"] = tmpVer - - assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err) -} - -func TestMultusConfigWithOverriddenName(t *testing.T) { - newNetworkName := "mega-net-2000" - multusConfig, _ := newMultusConfigWithDelegates( - primaryCNIName, - cniVersion, - kubeconfig, - primaryCNIConfig, - WithOverriddenName(newNetworkName)) - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}" - newTestCase(t, multusConfig.Generate).assertResult(expectedResult) -} - -func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase { - return &testCase{ - t: t, - configGenerationFunction: configGenerationFunc, - } -} - -func (tc testCase) assertResult(expectedResult string) { - multusCNIConfig, err := tc.configGenerationFunction() - if err != nil { - tc.t.Fatalf("error generating multus configuration: %v", err) - } - if multusCNIConfig != expectedResult { - tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig) - } -} +}) func documentHelper(pluginInfo string) interface{} { dp, _ := documentCNIData([]byte(pluginInfo)) diff --git a/pkg/server/config/manager.go b/pkg/server/config/manager.go index f4ef3beeb..b82a3af81 100644 --- a/pkg/server/config/manager.go +++ b/pkg/server/config/manager.go @@ -41,8 +41,6 @@ type Manager struct { multusConfigDir string multusConfigFilePath string primaryCNIConfigPath string - cniVersion string - forceCNIVersion bool } // NewManager returns a config manager object, configured to persist the @@ -65,19 +63,51 @@ func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigD return newManager(config, multusAutoconfigDir, primaryCNIPluginName, forceCNIVersion) } -func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) { +// overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case +func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error { + masterCNIConfigData, err := ioutil.ReadFile(cniConfigFile) + if err != nil { + return fmt.Errorf("failed to read cni config %s: %v", cniConfigFile, err) + } + + var primaryCNIConfigData map[string]interface{} + if err := json.Unmarshal(masterCNIConfigData, &primaryCNIConfigData); err != nil { + return fmt.Errorf("failed to unmarshall cni config %s: %w", cniConfigFile, err) + } + + primaryCNIConfigData["cniVersion"] = multusCNIVersion + configBytes, err := json.Marshal(primaryCNIConfigData) + if err != nil { + return fmt.Errorf("couldn't update cluster network config: %v", err) + } + + err = ioutil.WriteFile(cniConfigFile, configBytes, 0644) + if err != nil { + return fmt.Errorf("couldn't update cluster network config: %v", err) + } + return nil +} + +func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) { + if forceCNIVersion { + overrideCNIVersion(cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), config.CNIVersion) + } + watcher, err := newWatcher(multusConfigDir) if err != nil { return nil, err } + if defaultCNIPluginName == fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) { + return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", multusConfigDir, multusConfigFileName) + } + configManager := &Manager{ configWatcher: watcher, multusConfig: &config, multusConfigDir: multusConfigDir, multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName), primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), - forceCNIVersion: forceCNIVersion, } if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil { @@ -92,6 +122,11 @@ func (m *Manager) loadPrimaryCNIConfigFromFile() error { if err != nil { return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err) } + + if err = CheckVersionCompatibility(m.multusConfig, primaryCNIConfigData); err != nil { + return err + } + return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData) } @@ -115,7 +150,7 @@ func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface m.cniConfigData = cniConfigData return m.multusConfig.Mutate( - withDelegates(cniConfigData, m.multusConfig.CNIVersion, m.forceCNIVersion), + withClusterNetwork(m.primaryCNIConfigPath), withCapabilities(cniConfigData)) } @@ -128,10 +163,10 @@ func (m Manager) GenerateConfig() (string, error) { return m.multusConfig.Generate() } -// MonitorDelegatedPluginConfiguration monitors the configuration file pointed +// MonitorPluginConfiguration monitors the configuration file pointed // to by the primaryCNIPluginName attribute, and re-generates the multus // configuration whenever the primary CNI config is updated. -func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error { +func (m Manager) MonitorPluginConfiguration(shutDown chan struct{}, done chan struct{}) error { logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath) for { @@ -143,6 +178,7 @@ func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, don logging.Debugf("skipping un-related event %v", event) continue } + logging.Debugf("process event: %v", event) if !shouldRegenerateConfig(event) { continue diff --git a/pkg/server/config/manager_test.go b/pkg/server/config/manager_test.go index 592c5c905..abcf2c159 100644 --- a/pkg/server/config/manager_test.go +++ b/pkg/server/config/manager_test.go @@ -19,20 +19,12 @@ import ( "fmt" "io/ioutil" "os" - "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -const suiteName = "Configuration Manager" - -func TestMultusConfigurationManager(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, suiteName) -} - -var _ = Describe(suiteName, func() { +var _ = Describe("Configuration Manager", func() { const ( primaryCNIPluginName = "00-mycni.conf" primaryCNIPluginTemplate = ` @@ -55,17 +47,13 @@ var _ = Describe(suiteName, func() { multusConfigDir, err = ioutil.TempDir("", "multus-config") Expect(err).ToNot(HaveOccurred()) Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed()) - }) - BeforeEach(func() { defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName) Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed()) multusConf, _ := NewMultusConfig( primaryCNIName, - cniVersion, - kubeconfig) - var err error + cniVersion) configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false) Expect(err).NotTo(HaveOccurred()) }) @@ -75,52 +63,19 @@ var _ = Describe(suiteName, func() { }) It("Generates a configuration, based on the contents of the delegated CNI config file", func() { - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}" + expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"multus-cni-network\",\"clusterNetwork\":\"%s\",\"type\":\"myCNI\"}", defaultCniConfig) config, err := configManager.GenerateConfig() Expect(err).NotTo(HaveOccurred()) Expect(config).To(Equal(expectedResult)) }) - Context("Updates to the delegate CNI configuration", func() { - var ( - doneChannel chan struct{} - stopChannel chan struct{} - ) - - BeforeEach(func() { - doneChannel = make(chan struct{}) - stopChannel = make(chan struct{}) - go func() { - Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed()) - }() - }) - - AfterEach(func() { - go func() { stopChannel <- struct{}{} }() - Eventually(<-doneChannel).Should(Equal(struct{}{})) - close(doneChannel) - close(stopChannel) - }) - - It("Trigger the re-generation of the Multus CNI configuration", func() { - newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}" - Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), UserRWPermission)).To(Succeed()) - - multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) - Eventually(func() (string, error) { - multusCniData, err := ioutil.ReadFile(multusCniConfigFile) - return string(multusCniData), err - }).Should(Equal(multusConfigFromDelegate(newCNIConfig))) - }) - }) - When("the user requests the name of the multus configuration to be overridden", func() { BeforeEach(func() { Expect(configManager.OverrideNetworkName()).To(Succeed()) }) It("Overrides the name of the multus configuration when requested", func() { - expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}" + expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"mycni-name\",\"clusterNetwork\":\"%s\",\"type\":\"myCNI\"}", defaultCniConfig) config, err := configManager.GenerateConfig() Expect(err).NotTo(HaveOccurred()) Expect(config).To(Equal(expectedResult)) @@ -128,6 +83,41 @@ var _ = Describe(suiteName, func() { }) }) -func multusConfigFromDelegate(delegateConfig string) string { - return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig) +var _ = Describe("Configuration Manager with mismatched cniVersion", func() { + const ( + primaryCNIPluginName = "00-mycni.conf" + primaryCNIPluginTemplate = ` +{ + "cniVersion": "0.3.1", + "name": "mycni-name", + "type": "mycni", + "ipam": {}, + "dns": {} } +` + ) + + var multusConfigDir string + var defaultCniConfig string + + It("test cni version incompatibility", func() { + var err error + multusConfigDir, err = ioutil.TempDir("", "multus-config") + Expect(err).ToNot(HaveOccurred()) + Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed()) + + defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName) + Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed()) + + multusConf, _ := NewMultusConfig( + primaryCNIName, + cniVersion) + _, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false) + Expect(err).To(MatchError("failed to load the primary CNI configuration as a multus delegate with error 'delegate cni version is 0.3.1 while top level cni version is 0.4.0'")) + }) + + AfterEach(func() { + Expect(os.RemoveAll(multusConfigDir)).To(Succeed()) + }) + +}) diff --git a/pkg/server/server.go b/pkg/server/server.go index f86ddf090..ea9c1b219 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,6 +16,7 @@ package server import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -30,10 +31,10 @@ import ( cni100 "github.com/containernetworking/cni/pkg/types/100" "github.com/gorilla/mux" - "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config" k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/config" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types" ) @@ -76,25 +77,33 @@ func GetListener(socketPath string) (net.Listener, error) { } // NewCNIServer creates and returns a new Server object which will listen on a socket in the given path -func NewCNIServer(rundir string) (*Server, error) { +func NewCNIServer(rundir string, serverConfig []byte) (*Server, error) { kubeClient, err := k8s.InClusterK8sClient() if err != nil { return nil, fmt.Errorf("error getting k8s client: %v", err) } - return newCNIServer(rundir, kubeClient, nil) + return newCNIServer(rundir, kubeClient, nil, serverConfig) } -func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec) (*Server, error) { +func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) { + + // preprocess server config to be used to override multus CNI config + // see extractCniData() for the detail + if servConfig != nil { + servConfig = bytes.Replace(servConfig, []byte("{"), []byte(","), 1) + } + router := mux.NewRouter() s := &Server{ Server: http.Server{ Handler: router, }, - rundir: rundir, - requestFunc: HandleCNIRequest, - kubeclient: kubeClient, - exec: exec, + rundir: rundir, + requestFunc: HandleCNIRequest, + kubeclient: kubeClient, + exec: exec, + serverConfig: servConfig, } router.NotFoundHandler = http.HandlerFunc(http.NotFound) @@ -124,7 +133,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) { if err := json.Unmarshal(b, &cr); err != nil { return nil, err } - cmdType, cniCmdArgs, err := extractCniData(&cr) + cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig) if err != nil { return nil, fmt.Errorf("could not extract the CNI command args: %w", err) } @@ -142,7 +151,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) { return result, nil } -func extractCniData(cniRequest *Request) (string, *skel.CmdArgs, error) { +func extractCniData(cniRequest *Request, overrideConf []byte) (string, *skel.CmdArgs, error) { cmd, ok := cniRequest.Env["CNI_COMMAND"] if !ok { return "", nil, fmt.Errorf("unexpected or missing CNI_COMMAND") @@ -168,7 +177,20 @@ func extractCniData(cniRequest *Request) (string, *skel.CmdArgs, error) { return "", nil, fmt.Errorf("missing CNI_ARGS") } cniCmdArgs.Args = cniArgs - cniCmdArgs.StdinData = cniRequest.Config + + if overrideConf != nil { + // trim the close bracket from multus CNI config and put the server config + // to override CNI config with server config. + // note: if there are two or more value in same key, then the + // latest one is used at golang json implementation + idx := bytes.LastIndex(cniRequest.Config, []byte("}")) + if idx == -1 { + return "", nil, fmt.Errorf("invalid CNI config") + } + cniCmdArgs.StdinData = append(cniRequest.Config[:idx], overrideConf...) + } else { + cniCmdArgs.StdinData = cniRequest.Config + } return cmd, cniCmdArgs, nil } @@ -277,6 +299,7 @@ func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, k } func serializeResult(result cnitypes.Result) ([]byte, error) { + // cni result is converted to latest here and decoded to specific cni version at multus-shim realResult, err := cni100.NewResultFromResult(result) if err != nil { return nil, fmt.Errorf("failed to generate the CNI result: %w", err) diff --git a/pkg/server/shim.go b/pkg/server/shim.go index bfba62575..681addb34 100644 --- a/pkg/server/shim.go +++ b/pkg/server/shim.go @@ -18,64 +18,64 @@ import ( ) const ( - defaultMultusRunDir = "/var/run/multus-cni/" + defaultMultusRunDir = "/var/run/multus/" ) // CmdAdd implements the CNI spec ADD command handler func CmdAdd(args *skel.CmdArgs) error { - response, err := postRequest(args) + response, cniVersion, err := postRequest(args) if err != nil { return err } logging.Verbosef("CmdAdd (shim): %v", *response.Result) - return cnitypes.PrintResult(response.Result, response.Result.CNIVersion) + return cnitypes.PrintResult(response.Result, cniVersion) } // CmdCheck implements the CNI spec CHECK command handler func CmdCheck(args *skel.CmdArgs) error { - response, err := postRequest(args) + response, cniVersion, err := postRequest(args) if err != nil { return err } logging.Verbosef("CmdCheck (shim): %v", *response.Result) - return cnitypes.PrintResult(response.Result, response.Result.CNIVersion) + return cnitypes.PrintResult(response.Result, cniVersion) } // CmdDel implements the CNI spec DEL command handler func CmdDel(args *skel.CmdArgs) error { - response, err := postRequest(args) + response, cniVersion, err := postRequest(args) if err != nil { return err } logging.Verbosef("CmdDel (shim): %v", *response.Result) - return cnitypes.PrintResult(response.Result, response.Result.CNIVersion) + return cnitypes.PrintResult(response.Result, cniVersion) } -func postRequest(args *skel.CmdArgs) (*Response, error) { +func postRequest(args *skel.CmdArgs) (*Response, string, error) { multusShimConfig, err := shimConfig(args.StdinData) if err != nil { - return nil, fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err) + return nil, "", fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err) } cniRequest, err := newCNIRequest(args) if err != nil { - return nil, err + return nil, multusShimConfig.CNIVersion, err } body, err := DoCNI("http://dummy/", cniRequest, SocketPath(multusShimConfig.MultusSocketDir)) if err != nil { - return nil, err + return nil, multusShimConfig.CNIVersion, err } response := &Response{} if err = json.Unmarshal(body, response); err != nil { err = fmt.Errorf("failed to unmarshal response '%s': %v", string(body), err) - return nil, err + return nil, multusShimConfig.CNIVersion, err } - return response, nil + return response, multusShimConfig.CNIVersion, nil } // Create and fill a Request with this Plugin's environment and stdin which diff --git a/pkg/server/socket.go b/pkg/server/socket.go index 743223b89..aab4d4f76 100644 --- a/pkg/server/socket.go +++ b/pkg/server/socket.go @@ -22,7 +22,7 @@ import ( ) const ( - serverSocketName = "multus-cni.sock" + serverSocketName = "multus.sock" fullReadWriteExecutePermissions = 0777 thickPluginSocketRunDirPermissions = 0700 ) diff --git a/pkg/server/thick_cni_test.go b/pkg/server/thick_cni_test.go index 0013e59e5..32fd8ceee 100644 --- a/pkg/server/thick_cni_test.go +++ b/pkg/server/thick_cni_test.go @@ -114,7 +114,61 @@ var _ = Describe(suiteName, func() { K8sClient = fakeK8sClient() Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed()) - cniServer, err = startCNIServer(thickPluginRunDir, K8sClient) + cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, nil) + Expect(err).NotTo(HaveOccurred()) + + netns, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + // the namespace and podUID parameters below are hard-coded in the generation function + Expect(prepareCNIEnv(netns.Path(), "test", podName, "testUID")).To(Succeed()) + Expect(createFakePod(K8sClient, podName)).To(Succeed()) + }) + + AfterEach(func() { + Expect(cniServer.Close()).To(Succeed()) + Expect(teardownCNIEnv()).To(Succeed()) + Expect(K8sClient.Client.CoreV1().Pods("test").Delete( + context.TODO(), podName, metav1.DeleteOptions{})) + }) + + It("ADD works successfully", func() { + Expect(CmdAdd(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed()) + }) + + It("DEL works successfully", func() { + Expect(CmdDel(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed()) + }) + + It("CHECK works successfully", func() { + Expect(CmdCheck(cniCmdArgs(containerID, netns.Path(), ifaceName, referenceConfig(thickPluginRunDir)))).To(Succeed()) + }) + }) + + Context("CNI operations started from the shim with CNI config override with server config", func() { + const ( + containerID = "123456789" + ifaceName = "eth0" + podName = "my-little-pod" + ) + + var ( + cniServer *Server + K8sClient *k8s.ClientInfo + netns ns.NetNS + ) + + BeforeEach(func() { + var err error + K8sClient = fakeK8sClient() + + dummyServerConfig := `{ + "dummy_key1": "dummy_val1", + "dummy_key2": "dummy_val2" + }` + + Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed()) + cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, []byte(dummyServerConfig)) Expect(err).NotTo(HaveOccurred()) netns, err = testutils.NewNS() @@ -204,10 +258,10 @@ func createFakePod(k8sClient *k8s.ClientInfo, podName string) error { return err } -func startCNIServer(runDir string, k8sClient *k8s.ClientInfo) (*Server, error) { +func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) { const period = 0 - cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}) + cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig) if err != nil { return nil, err } @@ -228,6 +282,7 @@ func startCNIServer(runDir string, k8sClient *k8s.ClientInfo) (*Server, error) { func referenceConfig(thickPluginSocketDir string) string { const referenceConfigTemplate = `{ + "cniVersion": "0.4.0", "name": "node-cni-network", "type": "multus", "socketDir": "%s", diff --git a/pkg/server/types.go b/pkg/server/types.go index 3406d277f..a06c99f3f 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -25,10 +25,11 @@ type Request struct { // the CNI shim requests issued when a pod is added / removed. type Server struct { http.Server - requestFunc cniRequestFunc - rundir string - kubeclient *k8sclient.ClientInfo - exec invoke.Exec + requestFunc cniRequestFunc + rundir string + kubeclient *k8sclient.ClientInfo + exec invoke.Exec + serverConfig []byte } // Response represents the response (computed in the CNI server) for diff --git a/pkg/types/conf.go b/pkg/types/conf.go index cae971ea9..892c67cc2 100644 --- a/pkg/types/conf.go +++ b/pkg/types/conf.go @@ -44,7 +44,7 @@ const ( const ( // DefaultMultusDaemonConfigFile is the default path of the config file DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json" - defaultMultusRunDir = "/var/run/multus-cni/" + defaultMultusRunDir = "/var/run/multus/" ) // LoadDelegateNetConfList reads DelegateNetConf from bytes @@ -213,26 +213,29 @@ func NewCNIRuntimeConf(containerID, sandboxID, podName, podNamespace, podUID, ne // get CNI_ARGS and set it if it does not exist in rt.Args cniArgs := os.Getenv("CNI_ARGS") if cniArgs != "" { + logging.Debugf("ARGS: %s", cniArgs) for _, arg := range strings.Split(cniArgs, ";") { - for _, keyval := range strings.Split(arg, "=") { - if len(keyval) != 2 { - logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %s %s %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval)) - continue - } + logging.Debugf("arg: /%v/", arg) - envKey := string(keyval[0]) - envVal := string(keyval[1]) - isExists := false - for _, rtArg := range rt.Args { - if rtArg[0] == envKey { - isExists = true - } - } - if isExists != false { - logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg) - rt.Args = append(rt.Args, [2]string{envKey, envVal}) + keyval := strings.Split(arg, "=") + logging.Debugf("arg: /%q/, keyval: /%q/", arg, keyval) + if len(keyval) != 2 { + logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %v %v %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval)) + continue + } + + envKey := string(keyval[0]) + envVal := string(keyval[1]) + isExists := false + for _, rtArg := range rt.Args { + if rtArg[0] == envKey { + isExists = true } } + if isExists != false { + logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg) + rt.Args = append(rt.Args, [2]string{envKey, envVal}) + } } } @@ -357,7 +360,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { // the existing delegate list and all delegates executed in-order. if len(netconf.RawDelegates) == 0 && netconf.ClusterNetwork == "" { - return nil, logging.Errorf("LoadNetConf: at least one delegate/defaultNetwork must be specified") + return nil, logging.Errorf("LoadNetConf: at least one delegate/clusterNetwork must be specified") } if netconf.CNIDir == "" { @@ -424,15 +427,15 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { } // LoadDaemonNetConf loads the configuration for the multus daemon -func LoadDaemonNetConf(configPath string) (*ControllerNetConf, error) { +func LoadDaemonNetConf(configPath string) (*ControllerNetConf, []byte, error) { config, err := ioutil.ReadFile(configPath) if err != nil { - return nil, fmt.Errorf("failed to read the config file's contents: %w", err) + return nil, nil, fmt.Errorf("failed to read the config file's contents: %w", err) } daemonNetConf := &ControllerNetConf{} if err := json.Unmarshal(config, daemonNetConf); err != nil { - return nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err) + return nil, nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err) } logging.SetLogStderr(daemonNetConf.LogToStderr) @@ -459,7 +462,7 @@ func LoadDaemonNetConf(configPath string) (*ControllerNetConf, error) { daemonNetConf.MultusSocketDir = defaultMultusRunDir } - return daemonNetConf, nil + return daemonNetConf, config, nil } // AddDelegates appends the new delegates to the delegates list diff --git a/pkg/types/types.go b/pkg/types/types.go index dea57f172..8fa3a6ecd 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -30,21 +30,24 @@ type NetConf struct { // support chaining for master interface and IP decisions // occurring prior to running ipvlan plugin RawPrevResult *map[string]interface{} `json:"prevResult"` - PrevResult *cni100.Result `json:"-"` + PrevResult *cni100.Result `json:"-"` ConfDir string `json:"confDir"` CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - Kubeconfig string `json:"kubeconfig"` - ClusterNetwork string `json:"clusterNetwork"` - DefaultNetworks []string `json:"defaultNetworks"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` - LogToStderr bool `json:"logToStderr,omitempty"` - RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"` + RawDelegates []map[string]interface{} `json:"delegates"` + // These parameters are exclusive in one config file: + // - Delegates (directly add delegate CNI config into multus CNI config) + // - ClusterNetwork+DefaultNetworks (add CNI config through CRD, directory or file) + Delegates []*DelegateNetConf `json:"-"` + ClusterNetwork string `json:"clusterNetwork"` + DefaultNetworks []string `json:"defaultNetworks"` + Kubeconfig string `json:"kubeconfig"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + LogToStderr bool `json:"logToStderr,omitempty"` + RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"` // Default network readiness options ReadinessIndicatorFile string `json:"readinessindicatorfile"` // Option to isolate the usage of CR's to the namespace in which a pod resides.