Fix multus to support CNI plugin which does not create interface (#1096)

This change fixes multus to support CNI plugin which does not
create any interface and return empty result. Some CNI plugin
may do network configuration change only and does not create
any interface and return empty CNI result. Current multus assumes
that CNI config always creates some interfaces hence above CNI
plugin is out of assumption and multus may not work with such
plugins.

Co-authored-by: Tomofumi Hayashi <tohayash@redhat.com>

.github/workflows/build.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
+        go-version: [1.17.x, 1.18.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}

.github/workflows/image-build.yml

@@ -1,8 +1,26 @@
 name: Image build
 on: [pull_request]
 jobs:
+  ep-build-amd64:
+    name: Image build/amd64 LEGACY entrypoint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:ep-latest-amd64
+          file: images/Dockerfile
+
   build-amd64:
-    name: Image build/amd64
+    name: Image build/amd64 daemonized alternative
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
@@ -17,7 +35,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-amd64
-          file: deployments/Dockerfile
+          file: images/Dockerfile.thick
 
   build-arm64:
     name: Image build/arm64
@@ -35,7 +53,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-arm64
-          file: deployments/Dockerfile.arm64
+          file: images/Dockerfile.arm64
 
   build-arm32:
     name: Image build/arm32
@@ -53,7 +71,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-arm32
-          file: deployments/Dockerfile.arm32
+          file: images/Dockerfile.arm32
 
   build-ppc64le:
     name: Image build/ppc64le
@@ -71,7 +89,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-ppc64le
-          file: deployments/Dockerfile.ppc64le
+          file: images/Dockerfile.ppc64le
 
   build-s390:
     name: Image build/s390x
@@ -89,22 +107,4 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-s390x
-          file: deployments/Dockerfile.s390x
-
-  build-origin:
-    name: Image build/origin
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-origin
-          file: deployments/Dockerfile.openshift
+          file: images/Dockerfile.s390x

.github/workflows/image-push-master.yml

@@ -31,7 +31,17 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-amd64
             ghcr.io/${{ github.repository }}:snapshot-amd64
-          file: deployments/Dockerfile
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:thick-amd64
+          file: images/Dockerfile.thick
 
   push-arm64:
     name: Image push/arm64
@@ -60,7 +70,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-arm64
             ghcr.io/${{ github.repository }}:snapshot-arm64
-          file: deployments/Dockerfile.arm64
+          file: images/Dockerfile.arm64
 
   push-arm32:
     name: Image push/arm32
@@ -89,7 +99,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-arm32
             ghcr.io/${{ github.repository }}:snapshot-arm32
-          file: deployments/Dockerfile.arm32
+          file: images/Dockerfile.arm32
 
   push-ppc64le:
     name: Image push/ppc64le
@@ -118,7 +128,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-ppc64le
             ghcr.io/${{ github.repository }}:snapshot-ppc64le
-          file: deployments/Dockerfile.ppc64le
+          file: images/Dockerfile.ppc64le
 
   push-s390x:
     name: Image push/s390x
@@ -147,7 +157,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-s390x
             ghcr.io/${{ github.repository }}:snapshot-s390x
-          file: deployments/Dockerfile.s390x
+          file: images/Dockerfile.s390x
 
   push-origin:
     name: Image push/origin
@@ -176,7 +186,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest-origin
             ghcr.io/${{ github.repository }}:snapshot-origin
-          file: deployments/Dockerfile.openshift
+          file: images/Dockerfile.openshift
 
   push-manifest:
     needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
@@ -199,6 +209,11 @@ jobs:
         if: github.repository_owner == 'k8snetworkplumbingwg'
         run: |
           # get artifacts from previous steps
+          docker pull ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest create ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64 --arch amd64
+          docker manifest push ${{ env.REPOSITORY }}:thick
+
           docker pull ${{ env.REPOSITORY }}:snapshot-amd64
           docker pull ${{ env.REPOSITORY }}:snapshot-arm64
           docker pull ${{ env.REPOSITORY }}:snapshot-arm32

.github/workflows/image-push-release.yml

@@ -38,7 +38,18 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-amd64
             ${{ steps.docker_meta.outputs.tags }}-amd64
-          file: deployments/Dockerfile
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-thick-amd64
+            ${{ steps.docker_meta.outputs.tags }}-thick-amd64
+          file: images/Dockerfile.thick
 
   push-arm64:
     name: Image push/arm64
@@ -74,7 +85,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-arm64
             ${{ steps.docker_meta.outputs.tags }}-arm64
-          file: deployments/Dockerfile.arm64
+          file: images/Dockerfile.arm64
 
   push-arm32:
     name: Image push/arm32
@@ -110,7 +121,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-arm32
             ${{ steps.docker_meta.outputs.tags }}-arm32
-          file: deployments/Dockerfile.arm32
+          file: images/Dockerfile.arm32
 
   push-ppc64le:
     name: Image push/ppc64le
@@ -146,7 +157,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-ppc64le
             ${{ steps.docker_meta.outputs.tags }}-ppc64le
-          file: deployments/Dockerfile.ppc64le
+          file: images/Dockerfile.ppc64le
 
   push-s390x:
     name: Image push/s390x
@@ -182,7 +193,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-s390x
             ${{ steps.docker_meta.outputs.tags }}-s390x
-          file: deployments/Dockerfile.s390x
+          file: images/Dockerfile.s390x
 
   push-origin:
     name: Image push/origin
@@ -218,7 +229,7 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:stable-origin
             ${{ steps.docker_meta.outputs.tags }}-origin
-          file: deployments/Dockerfile.openshift
+          file: images/Dockerfile.openshift
 
   push-manifest:
     needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]

.github/workflows/kind-e2e.yml

@@ -14,7 +14,7 @@ jobs:
         run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
   
       - name: Build latest-amd64
-        run: docker build -t localhost:5000/multus:e2e -f deployments/Dockerfile .
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile.thick .
  
       - name: Push to local registry
         run: docker push localhost:5000/multus:e2e
@@ -35,6 +35,10 @@ jobs:
         working-directory: ./e2e
         run: ./test-simple-macvlan1.sh
 
+      - name: Test static pod
+        working-directory: ./e2e
+        run: ./test-static-pod.sh
+
       - name: Test default route1
         working-directory: ./e2e
         run: ./test-default-route1.sh

.github/workflows/legacy-kind-e2e.yml

@@ -0,0 +1,46 @@
+name: e2e-kind legacy installation with entrypoint script
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile .
+
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: MULTUS_MANIFEST=legacy-multus-daemonset.yml ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15.x
+          go-version: 1.17.x
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -16,11 +16,10 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - name: Run Revive
-      run: |
-        GO111MODULE=off go get github.com/mgechev/revive
-        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./... # this is ouput for user
-        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./...| xargs -0 -r false # this is for github actions
+    - name: Run Revive Action by pulling pre-built image
+      uses: docker://morphy/revive-action:v2
+      with:
+        exclude: "./vendor/..."
 
     - name: Run go fmt
       run: go fmt ./...

.goreleaser.yml

@@ -18,6 +18,8 @@ builds:
   - arm
   - arm64
   - s390x
+  ldflags:
+  - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date={{ .Date }}
 archives:
   - wrap_in_directory: true
 checksum:

README.md

@@ -27,7 +27,7 @@ The quickstart installation method for Multus requires that you have first insta
 Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)

cmd/config-generation/kubeconfig.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this generates kubeconfig file for multus based on service account
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+const userRWPermission = 0600
+
+const (
+	cniConfigDirVarName   = "cni-config-dir"
+	k8sCAFilePathVarName  = "kube-ca-file"
+	k8sServiceHostVarName = "k8s-service-host"
+	k8sServicePortVarName = "k8s-service-port"
+	serviceAccountPath    = "/var/run/secrets/kubernetes.io/serviceaccount"
+	skipTLSVerifyVarName  = "skip-tls-verify"
+)
+
+const (
+	defaultCniConfigDir   = "/host/etc/cni/net.d"
+	defaultK8sCAFilePath  = ""
+	defaultK8sServiceHost = ""
+	defaultK8sServicePort = 0
+	defaultSkipTLSValue   = false
+)
+
+func main() {
+	k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
+	k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
+	skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
+	kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	flag.Parse()
+
+	if *k8sServiceHost == defaultK8sServiceHost {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *k8sServicePort == defaultK8sServicePort {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *kubeCAFilePath == defaultK8sServiceHost {
+		*kubeCAFilePath = serviceAccountPath + "/ca.crt"
+	}
+
+	tlsCfg := "insecure-skip-tls-verify: true"
+	if !*skipTLSVerify {
+		kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
+		if err != nil {
+			logError("failed grabbing CA file: %w", err)
+		}
+		tlsCfg = "certificate-authority-data: " + kubeCAFileContents
+	}
+
+	multusConfigDir := *cniConfigDir + "/multus.d/"
+	if err := prepareCNIConfigDir(multusConfigDir); err != nil {
+		logError("failed to create CNI config dir: %w", err)
+	}
+	kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
+	serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
+	if err != nil {
+		logError("failed grabbing k8s token: %w", err)
+	}
+	if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
+		logError("failed generating kubeconfig: %w", err)
+	}
+}
+
+func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
+	data, err := ioutil.ReadFile(pathCAFile)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
+	}
+	return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
+}
+
+func k8sKubeConfigToken(tokenPath string) (string, error) {
+	data, err := ioutil.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
+	}
+	return string(data), nil
+}
+
+func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
+	kubeConfigTemplate := `
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: %s://[%s]:%d
+    %s
+users:
+- name: multus
+  user:
+    token: "%s"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+	kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
+	logInfo("Generated KubeConfig saved to %s: \n%s", outputPath, kubeconfig)
+	return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
+}
+
+func prepareCNIConfigDir(cniConfigDirPath string) error {
+	return os.MkdirAll(cniConfigDirPath, userRWPermission)
+}
+
+func logInvalidArg(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func logError(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	os.Exit(1)
+}
+
+func logInfo(format string, values ...interface{}) {
+	log.Printf("INFO: %s", fmt.Sprintf(format, values...))
+}

cmd/controller/main.go

@@ -0,0 +1,253 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this is daemonized entrypoint process. which watches master config
+// and generate multus CNI config
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+const (
+	multusPluginName     = "multus"
+	multusConfigFileName = "00-multus.conf"
+)
+
+const (
+	defaultCniConfigDir                 = "/etc/cni/net.d"
+	defaultMultusAdditionalBinDir       = ""
+	defaultMultusCNIVersion             = ""
+	defaultMultusConfigFile             = "auto"
+	defaultMultusGlobalNamespaces       = ""
+	defaultMultusKubeconfigPath         = "/etc/cni/net.d/multus.d/multus.kubeconfig"
+	defaultMultusLogFile                = ""
+	defaultMultusLogMaxSize             = 100 // megabytes
+	defaultMultusLogMaxAge              = 5   // days
+	defaultMultusLogMaxBackups          = 5
+	defaultMultusLogCompress            = true
+	defaultMultusLogLevel               = ""
+	defaultMultusLogToStdErr            = false
+	defaultMultusMasterCNIFile          = ""
+	defaultMultusNamespaceIsolation     = false
+	defaultMultusReadinessIndicatorFile = ""
+)
+
+const (
+	cniConfigDirVarName           = "cni-config-dir"
+	multusAdditionalBinDirVarName = "additional-bin-dir"
+	multusAutoconfigDirVarName    = "multus-autoconfig-dir"
+	multusCNIVersion              = "cni-version"
+	multusConfigFileVarName       = "multus-conf-file"
+	multusGlobalNamespaces        = "global-namespaces"
+	multusLogFile                 = "multus-log-file"
+	multusLogMaxSize              = "multus-log-max-size"
+	multusLogMaxAge               = "multus-log-max-age"
+	multusLogMaxBackups           = "multus-log-max-backups"
+	multusLogCompress             = "multus-log-compress"
+	multusLogLevel                = "multus-log-level"
+	multusLogToStdErr             = "multus-log-to-stderr"
+	multusKubeconfigPath          = "multus-kubeconfig-file-host"
+	multusMasterCNIFileVarName    = "multus-master-cni-file"
+	multusNamespaceIsolation      = "namespace-isolation"
+	multusReadinessIndicatorFile  = "readiness-indicator-file"
+)
+
+func main() {
+	versionOpt := false
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	multusConfigFile := flag.String(multusConfigFileVarName, defaultMultusConfigFile, "The multus configuration file to use. By default, a new configuration is generated.")
+	multusMasterCni := flag.String(multusMasterCNIFileVarName, defaultMultusMasterCNIFile, "The relative name of the configuration file of the cluster primary CNI.")
+	multusAutoconfigDir := flag.String(multusAutoconfigDirVarName, *cniConfigDir, "The directory path for the generated multus configuration.")
+	namespaceIsolation := flag.Bool(multusNamespaceIsolation, defaultMultusNamespaceIsolation, "If the network resources are only available within their defined namespaces.")
+	globalNamespaces := flag.String(multusGlobalNamespaces, defaultMultusGlobalNamespaces, "Comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled.")
+	logToStdErr := flag.Bool(multusLogToStdErr, defaultMultusLogToStdErr, "If the multus logs are also to be echoed to stderr.")
+	logLevel := flag.String(multusLogLevel, defaultMultusLogLevel, "One of: debug/verbose/error/panic. Used only with --multus-conf-file=auto.")
+	logFile := flag.String(multusLogFile, defaultMultusLogFile, "Path where to multus will log. Used only with --multus-conf-file=auto.")
+	logMaxSize := flag.Int(multusLogMaxSize, defaultMultusLogMaxSize, "the maximum size in megabytes of the log file before it gets rotated")
+	logMaxAge := flag.Int(multusLogMaxAge, defaultMultusLogMaxAge, "the maximum number of days to retain old log files in their filename")
+	logMaxBackups := flag.Int(multusLogMaxBackups, defaultMultusLogMaxBackups, "the maximum number of old log files to retain")
+	logCompress := flag.Bool(multusLogCompress, defaultMultusLogCompress, "compress determines if the rotated log files should be compressed using gzip")
+	cniVersion := flag.String(multusCNIVersion, defaultMultusCNIVersion, "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
+	additionalBinDir := flag.String(multusAdditionalBinDirVarName, defaultMultusAdditionalBinDir, "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
+	readinessIndicator := flag.String(multusReadinessIndicatorFile, defaultMultusReadinessIndicatorFile, "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
+	multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
+	overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	if *logToStdErr {
+		logging.SetLogStderr(*logToStdErr)
+	}
+	if *logFile != defaultMultusLogFile {
+		logging.SetLogFile(*logFile)
+	}
+	if *logLevel != defaultMultusLogLevel {
+		logging.SetLogLevel(*logLevel)
+	}
+
+	if *multusConfigFile == defaultMultusConfigFile {
+		if *cniVersion == defaultMultusCNIVersion {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		var configurationOptions []config.Option
+		if *namespaceIsolation {
+			configurationOptions = append(
+				configurationOptions, config.WithNamespaceIsolation())
+		}
+
+		if *globalNamespaces != defaultMultusGlobalNamespaces {
+			configurationOptions = append(
+				configurationOptions, config.WithGlobalNamespaces(*globalNamespaces))
+		}
+
+		if *logToStdErr != defaultMultusLogToStdErr {
+			configurationOptions = append(
+				configurationOptions, config.WithLogToStdErr())
+		}
+
+		if *logLevel != defaultMultusLogLevel {
+			configurationOptions = append(
+				configurationOptions, config.WithLogLevel(*logLevel))
+		}
+
+		if *logFile != defaultMultusLogFile {
+			configurationOptions = append(
+				configurationOptions, config.WithLogFile(*logFile))
+		}
+
+		if *additionalBinDir != defaultMultusAdditionalBinDir {
+			configurationOptions = append(
+				configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
+		}
+
+		if *readinessIndicator != defaultMultusReadinessIndicatorFile {
+			configurationOptions = append(
+				configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
+		}
+
+		// logOptions
+
+		var logOptionFuncs []config.LogOptionFunc
+		if *logMaxAge != defaultMultusLogMaxAge {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxAge(logMaxAge))
+		}
+		if *logMaxSize != defaultMultusLogMaxSize {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxSize(logMaxSize))
+		}
+		if *logMaxBackups != defaultMultusLogMaxBackups {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxBackups(logMaxBackups))
+		}
+		if *logCompress != defaultMultusLogCompress {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogCompress(logCompress))
+		}
+
+		if len(logOptionFuncs) > 0 {
+			logOptions := &config.LogOptions{}
+			config.MutateLogOptions(logOptions, logOptionFuncs...)
+			configurationOptions = append(configurationOptions, config.WithLogOptions(logOptions))
+		}
+
+		multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
+		if err != nil {
+			_ = logging.Errorf("Failed to create multus config: %v", err)
+			os.Exit(3)
+		}
+
+		var configManager *config.Manager
+		if *multusMasterCni == "" {
+			configManager, err = config.NewManager(*multusConfig, *multusAutoconfigDir)
+		} else {
+			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
+				*multusConfig, *multusAutoconfigDir, *multusMasterCni)
+		}
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+
+		if *overrideNetworkName {
+			if err := configManager.OverrideNetworkName(); err != nil {
+				_ = logging.Errorf("could not override the network name: %v", err)
+			}
+		}
+
+		generatedMultusConfig, err := configManager.GenerateConfig()
+		if err != nil {
+			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
+		}
+		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
+			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+		}
+
+		configWatcherDoneChannel := make(chan struct{})
+		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
+			defer func() {
+				stopChannel <- struct{}{}
+			}()
+			if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
+				_ = logging.Errorf("error watching file: %v", err)
+			}
+		}(make(chan struct{}), configWatcherDoneChannel)
+
+		<-configWatcherDoneChannel
+	} else {
+		if err := copyUserProvidedConfig(*multusConfigFile, *cniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", *multusConfigFile, err)
+		}
+	}
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	srcFile, err := os.Open(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(srcFile, dstFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/main.go

@@ -15,7 +15,6 @@
 // This is a "Multi-plugin".The delegate concept referred from CNI project
 // It reads other plugin netconf, and then invoke them, e.g.
 // flannel or sriov plugin.
-
 package main
 
 import (

deployments/multus-daemonset-crio.yml

@@ -166,6 +166,8 @@ spec:
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
       serviceAccountName: multus
       containers:
       - name: kube-multus
@@ -191,6 +193,7 @@ spec:
         volumeMounts:
         - name: run
           mountPath: /run
+          mountPropagation: HostToContainer
         - name: cni
           mountPath: /host/etc/cni/net.d
         - name: cnibin

deployments/multus-daemonset-gke-1.16.yml

@@ -139,6 +139,8 @@ spec:
       tolerations:
         - operator: Exists
           effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
       serviceAccountName: multus
       containers:
         - name: kube-multus

deployments/multus-daemonset-thick-plugin.yml

@@ -0,0 +1,189 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+          args:
+            - "-cni-version=0.3.1"
+            - "-cni-config-dir=/host/etc/cni/net.d"
+            - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+            - "-multus-log-to-stderr=true"
+            - "-multus-log-level=verbose"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+        - name: generate-kubeconfig
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "/usr/src/multus-cni/bin/generate-kubeconfig"
+          args:
+            - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+            - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+

deployments/multus-daemonset.yml

@@ -166,6 +166,8 @@ spec:
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
       serviceAccountName: multus
       containers:
       - name: kube-multus
@@ -190,6 +192,23 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
       terminationGracePeriodSeconds: 10
       volumes:
         - name: cni

docs/configuration.md

@@ -14,6 +14,12 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     "binDir": "/opt/cni/bin",
     "logFile": "/var/log/multus.log",
     "logLevel": "debug",
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    },
     "capabilities": {
         "portMappings": true
     },    
@@ -44,6 +50,7 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
 * `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
 * `logFile` (string, optional): file path for log file. multus puts log in given file
 * `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
+* `logOptions` (object, optional): logging option, More detailed log configuration
 * `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
 * `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
 * `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
@@ -103,7 +110,7 @@ Optionally, you may have Multus log to a file on the filesystem. This file will
 For example in your CNI configuration, you may set:
 
 ```
-    "LogFile": "/var/log/multus.log",
+    "logFile": "/var/log/multus.log",
 ```
 
 #### Logging Level
@@ -120,7 +127,27 @@ The available logging level values, in decreasing order of verbosity are:
 You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
 
 ```
-    "LogLevel": "debug",
+    "logLevel": "debug",
+```
+
+#### Logging Options
+
+If you want a more detailed configuration of the logging, This includes the following parameters:
+
+* `maxAge` the maximum number of days to retain old log files in their filename
+* `maxSize` the maximum size in megabytes of the log file before it gets rotated
+* `maxBackups` the maximum number of days to retain old log files in their filename
+* `compress` compress determines if the rotated log files should be compressed using gzip
+
+For example in your CNI configuration, you may set:
+
+```
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    }
 ```
 
 ### Namespace Isolation

docs/development.md

@@ -35,7 +35,7 @@ cd multus-cni
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
 ```
-sudo ./scripts/test.sh
+sudo ./hack/test-go.sh
 ```
 
 ## What are the best practices for logging?

docs/how-to-use.md

@@ -15,13 +15,13 @@ Generally we recommend two options: Manually place a Multus binary in your `/opt
 
 You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/k8snetworkplumbingwg/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
 
-    $ cp multus /opt/cni/bin
+    cp multus /opt/cni/bin
 
 *Via Daemonset method*
 
 As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    $ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
+    cat ./deployments/multus-daemonset.yml | kubectl apply -f -
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -34,8 +34,8 @@ You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alp
 Execute following commands at all Kubernetes nodes (i.e. master and minions)
 
 ```
-$ mkdir -p /etc/cni/net.d
-$ cat >/etc/cni/net.d/00-multus.conf <<EOF
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/00-multus.conf <<EOF
 {
   "name": "multus-cni-network",
   "type": "multus",
@@ -72,7 +72,7 @@ Create resources for multus to access CRD objects as following command:
 
 ```
 # Execute following commands at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -119,13 +119,13 @@ Create kubeconfig at master node as following commands:
 
 ```
 # Execute following command at Kubernetes master
-$ mkdir -p /etc/cni/net.d/multus.d
-$ SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
-$ SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-$ KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
-$ KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
-$ KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
-$ cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
+mkdir -p /etc/cni/net.d/multus.d
+SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
+SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
+KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
+KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
+cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
 # Kubeconfig file for Multus CNI plugin.
 apiVersion: v1
 kind: Config
@@ -151,7 +151,7 @@ Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
 **NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
 
 ```
-$ scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
+scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
 ```
 
 ### Setup CRDs (daemonset automatically does)
@@ -162,7 +162,7 @@ Create CRD definition in Kubernetes as following command at master node:
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -200,7 +200,7 @@ Following command creates NetworkAttachmentDefinition. CNI config is in `config:
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -232,7 +232,7 @@ If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -242,7 +242,7 @@ EOF
 
 ```
 # Execute following commands at all Kubernetes nodes (i.e. master and minions)
-$ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
+cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
 {
   "cniVersion": "0.3.0",
   "type": "macvlan",
@@ -268,7 +268,7 @@ $ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -290,7 +290,7 @@ You can also specify NetworkAttachmentDefinition with its namespace as adding `<
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -314,7 +314,7 @@ spec:
             }
         }'
 EOF
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -336,7 +336,7 @@ You can also specify interface name as adding `@<ifname>`.
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -356,7 +356,7 @@ EOF
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -381,7 +381,7 @@ You can also specify NetworkAttachmentDefinition with its namespace as adding `"
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -406,7 +406,7 @@ You can also specify interface name as adding `"interface": "<ifname>"`.
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -432,7 +432,8 @@ Following the example of `ip -d address` output of above pod, "pod-case-06":
 
 ```
 # Execute following command at Kubernetes master
-$ kubectl exec -it pod-case-06 -- ip -d address
+kubectl exec -it pod-case-06 -- ip -d address
+
 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
     inet 127.0.0.1/8 scope host lo
@@ -529,7 +530,8 @@ EOF
 This will set `192.168.2.1` as the default route over the `net1` interface, such as:
 
 ```
-$ kubectl exec -it samplepod -- ip route
+kubectl exec -it samplepod -- ip route
+
 default via 192.168.2.1 dev net1 
 10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
 10.244.0.0/16 via 10.244.0.1 dev eth0 
@@ -602,7 +604,7 @@ The `--multus-master-cni-file-name` can be used to select the cni file as the ma
     --multus-log-level=
     --multus-log-file=
 
-Used only with `--multus-conf-file=auto`. See the documentation for logging for which values are permitted.
+Used only with `--multus-conf-file=auto`. See the [documentation for logging](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#logging) for which values are permitted.
 
 Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to specify CNI spec version.
 

docs/quickstart.md

@@ -15,7 +15,7 @@ Two things we'll refer to a number of times through this document are:
 
 Our installation method requires that you first have installed Kubernetes and have configured a default network -- that is, a CNI plugin that's used for your pod-to-pod connectivity. 
 
-We recommend Kubernetes 1.16 or later.
+We support Kubernetes versions that Kubernetes community supports. Please see [Supported versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions) in Kubernetes document.
 
 To install Kubernetes, you may decide to use [kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/), or potentially [kubespray](https://github.com/kubernetes-sigs/kubespray).
 
@@ -51,13 +51,13 @@ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cn
 We'll apply a YAML file with `kubectl` from this repo.
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 ### What the Multus daemonset does
 
 * Starts a Multus daemonset, this runs a pod on each node which places a Multus binary on each node in `/opt/cni/bin`
-* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
+* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus on each node as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
 * Creates a `/etc/cni/net.d/multus.d` directory on each node with authentication information for Multus to access the Kubernetes API.
 
 
@@ -66,7 +66,7 @@ $ cat ./images/multus-daemonset.yml | kubectl apply -f -
 Generally, the first step in validating your installation is to ensure that the Multus pods have run without error, you may see an overview of those by looking at:
 
 ```
-$ kubectl get pods --all-namespaces | grep -i multus
+kubectl get pods --all-namespaces | grep -i multus
 ```
 
 You may further validate that it has ran by looking at the `/etc/cni/net.d/` directory and ensure that the auto-generated `/etc/cni/net.d/00-multus.conf` exists corresponding to the alphabetically first configuration file.
@@ -176,7 +176,7 @@ EOF
 You may now inspect the pod and see what interfaces are attached, like so:
 
 ```
-$ kubectl exec -it samplepod -- ip a
+kubectl exec -it samplepod -- ip a
 ```
 
 You should note that there are 3 interfaces:

e2e/cni-install.yml

@@ -7,9 +7,9 @@ metadata:
 data:
   install_cni.sh: |
     cd /tmp
-    wget https://github.com/containernetworking/plugins/releases/download/v0.8.5/cni-plugins-linux-amd64-v0.8.5.tgz
+    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
     cd /host/opt/cni/bin
-    tar xvfzp /tmp/cni-plugins-linux-amd64-v0.8.5.tgz
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
     sleep infinite
 ---
 apiVersion: apps/v1

e2e/get_tools.sh

@@ -5,11 +5,11 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.8.1/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl
-curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.82/koko_0.82_linux_amd64
+curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq

e2e/legacy-multus-daemonset.yml

@@ -0,0 +1,264 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/multus-daemonset.yml

@@ -44,6 +44,15 @@ rules:
     verbs:
       - get
       - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -145,10 +154,16 @@ spec:
       containers:
       - name: kube-multus
         image: localhost:5000/multus:e2e
-        command: ["/entrypoint.sh"]
+        imagePullPolicy: Always
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
         args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
+        - "-multus-conf-file=auto"
+        - "-cni-version=0.3.1"
+        - "-cni-config-dir=/host/etc/cni/net.d"
+        - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "-multus-log-to-stderr=true"
+        - "-multus-log-level=debug"
+        - "-multus-log-file=/tmp/multus.log"
         resources:
           requests:
             cpu: "100m"
@@ -165,6 +180,40 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      - name: generate-kubeconfig
+        image: localhost:5000/multus:e2e
+        command:
+          - "/usr/src/multus-cni/bin/generate-kubeconfig"
+        args:
+          - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+          - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cni
+            mountPath: /host/etc/cni/net.d
+            mountPropagation: Bidirectional
       volumes:
         - name: cni
           hostPath:

e2e/setup_cluster.sh

@@ -3,19 +3,28 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
+# define the OCI binary to be used. Acceptable values are `docker`, `podman`.
+# Defaults to `docker`.
+OCI_BIN="${OCI_BIN:-docker}"
+
+# define the deployment spec to use when deploying multus.
+# Acceptable values are `legacy-multus-daemonset.yml`. `multus-daemonset.yml`.
+# Defaults to `multus-daemonset.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset.yml}"
+
 kind_network='kind'
 reg_name='kind-registry'
 reg_port='5000'
-running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
 if [ "${running}" != 'true' ]; then
   # run registry and push the multus image
-  docker run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
-  docker build -t localhost:5000/multus:e2e -f ../deployments/Dockerfile ..
-  docker push localhost:5000/multus:e2e
+  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
+  $OCI_BIN push localhost:5000/multus:e2e
 fi
 reg_host="${reg_name}"
 if [ "${kind_network}" = "bridge" ]; then
-    reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
 fi
 echo "Registry Host: ${reg_host}"
 
@@ -30,6 +39,12 @@ containerdConfigPatches:
 nodes:
   - role: control-plane
   - role: worker
+    kubeadmConfigPatches:
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          pod-manifest-path: "/etc/kubernetes/manifests/"
   - role: worker
 EOF
 
@@ -45,7 +60,7 @@ data:
     help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
 EOF
 
-containers=$(docker network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
+containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
 needs_connect="true"
 for c in $containers; do
   if [ "$c" = "${reg_name}" ]; then
@@ -53,14 +68,17 @@ for c in $containers; do
   fi
 done
 if [ "${needs_connect}" = "true" ]; then
-  docker network connect "${kind_network}" "${reg_name}" || true
+  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
 fi
 
+worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
+worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
+
 kind export kubeconfig
-sudo env PATH=${PATH} koko -d kind-worker,eth1 -d kind-worker2,eth1
+sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
 sleep 1
 kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f multus-daemonset.yml
+kubectl create -f "$MULTUS_MANIFEST"
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
 kubectl create -f cni-install.yml

e2e/simple-static-pod.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web
+  annotations:
+    k8s.v1.cni.cncf.io/networks: "bridge-nad"
+spec: 
+  containers: 
+    - name: web 
+      image: centos:8 
+      command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]

e2e/static-pod-nad.yml

@@ -0,0 +1,15 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: bridge-nad
+spec: 
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "testnet",
+    "type": "bridge",
+    "bridge": "testnet0",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "10.10.0.0/16"
+    }
+}'

e2e/test-static-pod.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o errexit
+
+echo "Creating network attachment definition"
+kubectl create -f static-pod-nad.yml
+
+echo "Creating static pod config file"
+docker cp simple-static-pod.yml kind-worker:/etc/kubernetes/manifests/static-web.yaml
+
+echo "Waiting for static pod to start"
+kubectl wait --for=condition=Ready --namespace=default pod/static-web-kind-worker
+
+echo "Checking the pod annotation for net1 interface"
+kubectl exec static-web-kind-worker --namespace=default -- ip a show dev net1
+
+echo "Deleting static pod"
+docker exec kind-worker /bin/bash -c "rm /etc/kubernetes/manifests/static-web.yaml"
+
+echo "Deleting network attachment definition"
+kubectl delete -f static-pod-nad.yml
+
+echo "Test complete"

go.mod

@@ -1,16 +1,18 @@
 module gopkg.in/k8snetworkplumbingwg/multus-cni.v3
 
-go 1.16
+go 1.17
 
 require (
+	github.com/blang/semver v3.5.1+incompatible
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.9.1
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20221215110210-ad3f3381681f
 	github.com/onsi/ginkgo v1.12.1
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1
 	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
-	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	golang.org/x/net v0.6.0
 	google.golang.org/grpc v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	k8s.io/api v0.20.10
@@ -21,8 +23,56 @@ require (
 	k8s.io/kubernetes v1.20.10
 )
 
+require (
+	github.com/Microsoft/go-winio v0.4.15 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v4.9.0+incompatible // indirect
+	github.com/go-logr/logr v0.2.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nxadm/tail v1.4.4 // indirect
+	github.com/prometheus/client_golang v1.7.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.2.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+	k8s.io/apiserver v0.20.10 // indirect
+	k8s.io/component-base v0.20.10 // indirect
+	k8s.io/klog/v2 v2.4.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
 replace (
 	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	golang.org/x/net => golang.org/x/net v0.6.0
+	golang.org/x/text => golang.org/x/text v0.8.0
 	k8s.io/api => k8s.io/api v0.20.10
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.10
 	k8s.io/apimachinery => k8s.io/apimachinery v0.20.10

go.sum

@@ -24,12 +24,10 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
@@ -52,7 +50,6 @@ github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990/go.mod h1:ay/0dTb7NsG8QMDfsRfLHgZo/6xAJShLe1+ePPflihk=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -102,20 +99,16 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
 github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
 github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
-github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
 github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
-github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
@@ -130,7 +123,6 @@ github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ
 github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8=
 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
 github.com/coredns/corefile-migration v1.0.10/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI=
-github.com/coredns/corefile-migration v1.0.11/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -147,8 +139,6 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
@@ -162,10 +152,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -173,7 +161,6 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -181,7 +168,6 @@ github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQm
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
-github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
@@ -190,7 +176,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -201,7 +186,6 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
 github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
-github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -212,8 +196,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@@ -245,12 +227,10 @@ github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsd
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
-github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
@@ -259,10 +239,8 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
-github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -272,7 +250,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
@@ -282,12 +259,9 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
@@ -307,10 +281,7 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/cadvisor v0.38.8/go.mod h1:1OFB9sOOMkBdUBGCO/1SArawTnDscgMzTodacVDe8mA=
-github.com/google/cadvisor v0.39.0 h1:jai6dmBP9QAYluNGqU18yVUTw6uuyAW0AqtZIjvl8Qg=
-github.com/google/cadvisor v0.39.0/go.mod h1:rjQFmK4jPCpxeUdLq9bYhNFFsjgGOtpnDmDeap0+nsw=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -327,15 +298,12 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
@@ -367,7 +335,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -375,9 +342,7 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
-github.com/heketi/heketi v10.2.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
 github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
@@ -398,8 +363,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05 h1:vIZIhdbcm2eoTzSyaBQqgKLHzymSBbXpudf+2Wv4FNM=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20221215110210-ad3f3381681f h1:SVJhARCyy2mmDUegdEBnYyLqjB4h6BxsfN5rfehExuU=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20221215110210-ad3f3381681f/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -408,15 +373,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
@@ -434,13 +397,11 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
@@ -449,7 +410,6 @@ github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPx
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -462,12 +422,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
-github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
-github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
-github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -475,10 +431,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
-github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
-github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
@@ -486,46 +440,35 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE=
-github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.8.0 h1:+77ba4ar4jsCbL1GLbFL8fFM57w6suPfSS9PDLDY7KM=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -572,20 +515,17 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -607,13 +547,11 @@ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -632,13 +570,11 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3C
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -650,7 +586,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -667,11 +602,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -679,7 +613,6 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
@@ -687,7 +620,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20210220032938-85be41e4509f/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -703,59 +635,18 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
@@ -767,6 +658,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -793,12 +686,9 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -814,46 +704,28 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -882,10 +754,8 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -897,7 +767,8 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -922,14 +793,12 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -954,7 +823,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -978,11 +846,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -1002,18 +867,13 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1022,49 +882,26 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.20.10 h1:kAdgi1zcyenV88/uVEzS9B/fn1m4KRbmdKB0Lxl6z/M=
 k8s.io/api v0.20.10/go.mod h1:0kei3F6biGjtRQBo5dUeujq6Ji3UCh9aOSfp/THYd7I=
-k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
-k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
 k8s.io/apiextensions-apiserver v0.20.10/go.mod h1:am9XHHsM/FJBgPtl586TGSDAouRTLZC6wu25rb2VqCQ=
-k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
 k8s.io/apimachinery v0.20.10 h1:GcFwz5hsGgKLohcNgv8GrInk60vUdFgBXW7uOY1i1YM=
 k8s.io/apimachinery v0.20.10/go.mod h1:kQa//VOAwyVwJ2+L9kOREbsnryfsGSkSM1przND4+mw=
-k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
-k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
 k8s.io/apiserver v0.20.10 h1:9Th11BLOMY5HHHp2AciNNgp/y71XKO+FLHO0pultyrQ=
 k8s.io/apiserver v0.20.10/go.mod h1:bBm1wyFuID+0CsEYIdyiamOw9wEniEOq3HrCU40ky2M=
-k8s.io/apiserver v0.21.1 h1:wTRcid53IhxhbFt4KTrFSw8tAncfr01EP91lzfcygVg=
-k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
 k8s.io/cli-runtime v0.20.10/go.mod h1:O8xLwPPEJsTyiCB/ePt4JgEPbStQUzSD8rt7GJJj6Kw=
-k8s.io/cli-runtime v0.21.1/go.mod h1:TI9Bvl8lQWZB2KqE91QLCp9AZE4l29zNFnj/x4IX4Fw=
 k8s.io/client-go v0.20.10 h1:TgAL2pqcNWMH4eZoS9Uw0BLh2lu5a2A4pmegjp5pmsk=
 k8s.io/client-go v0.20.10/go.mod h1:fFg+aLoasv/R+xiVaWjxeqGFYltzgQcOQzkFaSRfnJ0=
-k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
-k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
 k8s.io/cloud-provider v0.20.10/go.mod h1:8HQ0NgW661PiH+QK1BPYD0QorpaxOXQs1ZsMfOe3uc0=
-k8s.io/cloud-provider v0.21.1 h1:V7ro0ZuxMBNYVH4lJKxCdI+h2bQ7EApC5f7sQYrQLVE=
-k8s.io/cloud-provider v0.21.1/go.mod h1:GgiRu7hOsZh3+VqMMbfLJJS9ZZM9A8k/YiZG8zkWpX4=
 k8s.io/cluster-bootstrap v0.20.10/go.mod h1:D+cqd8iJYeajaIUBUca4J3f/87L4qiFvMPzM5qs8x1o=
-k8s.io/cluster-bootstrap v0.21.1/go.mod h1:izdXmPTfqI9gkjQLf9PQ1Y9/DSqG54zU2UByEzgdajs=
 k8s.io/code-generator v0.20.10/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU=
-k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
 k8s.io/component-base v0.20.10 h1:QNlekT6M2zBb4feHHmZ+YHZHcDbhbrYS7xHHY+v+kOE=
 k8s.io/component-base v0.20.10/go.mod h1:ZKOEin1xu68aJzxgzl5DZSp5J1IrjAOPlPN90/t6OI8=
-k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
-k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
 k8s.io/component-helpers v0.20.10/go.mod h1:9SuOCO69yzUr8t9oajyO40NPAYK3JCYXwwyLS3YINR4=
-k8s.io/component-helpers v0.21.1 h1:jhi4lHGHOV6mbPqNfITVUoLC3kNFkBQQO1rDDpnThAw=
-k8s.io/component-helpers v0.21.1/go.mod h1:FtC1flbiQlosHQrLrRUulnKxE4ajgWCGy/67fT2GRlQ=
 k8s.io/controller-manager v0.20.10/go.mod h1:NwFcdJR5ZK0pjKNUZuZbus/tO8I0zSkGpp0ifQi2DK0=
-k8s.io/controller-manager v0.21.1/go.mod h1:8ugs8DCcHqybiwdVERhnnyGoS5Ksq/ea1p2B0CosHyc=
 k8s.io/cri-api v0.20.10/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
-k8s.io/cri-api v0.21.1 h1:dLQx77FHendlecUB9SE9GPls6clStVQ3eRsAzLIsr54=
-k8s.io/cri-api v0.21.1/go.mod h1:nJbXlTpXwYCYuGMR7v3PQb1Du4WOGj2I9085xMVjr3I=
 k8s.io/csi-translation-lib v0.20.10/go.mod h1:dVQvr/Y/74jFZU955V/KqgZJ4E4hRF4IcsxUq0WbUrc=
-k8s.io/csi-translation-lib v0.21.1/go.mod h1:y6NwcsxV1IsoqOm07G4hqANvCLXNDMC1t00lf+CqKRA=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
@@ -1073,44 +910,23 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-aggregator v0.20.10/go.mod h1:Y8T4ttD/sJIhaL84giy6aP9Q7aF9B23N/2u5KqEa/gQ=
-k8s.io/kube-aggregator v0.21.1/go.mod h1:cAZ0n02IiSl57sQSHz4vvrz3upQRMbytOiZnpPJaQzQ=
 k8s.io/kube-controller-manager v0.20.10/go.mod h1:6XCS/QLajausNkLu/fPe0Pj6TxbpuNd51BQTqSwBIbg=
-k8s.io/kube-controller-manager v0.21.1/go.mod h1:zEzQfcDGMQFFFpeWXv5GdJKIDR00LB4wp+hKYeRw7yc=
-k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY=
 k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kube-proxy v0.20.10/go.mod h1:hNgnO70MFOwZHQb87EYukEuy2uUvhrg/XZG2OTnyMy4=
-k8s.io/kube-proxy v0.21.1/go.mod h1:5/Vg0HZHf2+475YhaeXWPvrw7PcQF0hW1Kx3Ug6kyBI=
 k8s.io/kube-scheduler v0.20.10/go.mod h1:GYb7hAdtx2DZ3l8SLhtcS9Qm+U5lgjCJIgF/QYKr1s0=
-k8s.io/kube-scheduler v0.21.1 h1:knkUyAYE8i3v4hMdsLd7s2cHfhc+GjKOj3vzcBdInEM=
-k8s.io/kube-scheduler v0.21.1/go.mod h1:AFLD3tRlC0lbhB+WClraGOV/e5+NQyi5PeeL2CMKx/Y=
 k8s.io/kubectl v0.20.10/go.mod h1:nM2vIk+39DHuXDmdCL3AehkkhbGUgRAlyPgkfycHoXY=
-k8s.io/kubectl v0.21.1/go.mod h1:PMYR88MqESuysBM/MX+Vu4JbX/50nY4d4kny+SPEI2U=
 k8s.io/kubelet v0.20.10 h1:zyYFKuvCNVKVx1LWpAOwuYuZ7zdb9i/BI53Vy5rRdSw=
 k8s.io/kubelet v0.20.10/go.mod h1:OYBcBg22jbmycu0V6IMtq0Xpjoxj2iJ+LtH6XWsuesA=
-k8s.io/kubelet v0.21.1 h1:JeZsCr3GN2Kjg3gn21jLU10RFu0APUK/vdpFWa8P8Nw=
-k8s.io/kubelet v0.21.1/go.mod h1:poOR6Iaa5WqytFOp0egXFV8c2XTLFxaXTdj5njUlnVY=
 k8s.io/kubernetes v1.20.10 h1:4ng4jMVHQfLtUwD1Bv+xjzVPP2Zp41GeFVWK8x1Wwl0=
 k8s.io/kubernetes v1.20.10/go.mod h1:iE/QvEbLD6Ne9U03MX/BTBVSbj3pnj/HewBr8XDzLxw=
-k8s.io/kubernetes v1.21.1 h1:U7cVOSdG+sMNOfL9XlenBV7avSBDHyWPE66gWnnYIIc=
-k8s.io/kubernetes v1.21.1/go.mod h1:ef++isEL1PW0taH6z7DXrSztPglrZ7jQhyvcMEtm0gQ=
 k8s.io/legacy-cloud-providers v0.20.10/go.mod h1:5xDo9brr2bkusQr9KLWnUKY2bNZ52+9CJiIaqyGn7CE=
-k8s.io/legacy-cloud-providers v0.21.1/go.mod h1:rpyWrAsPV1YNn7MmkZydMqleJijhay1Eqx+32K06Rd0=
 k8s.io/metrics v0.20.10/go.mod h1:M0IJySt6DSTjZQZamJ70b+F7I12pXqaeQmv/VpexC9I=
-k8s.io/metrics v0.21.1/go.mod h1:pyDVLsLe++FIGDBFU80NcW4xMFsuiVTWL8Zfi7+PpNo=
 k8s.io/mount-utils v0.20.10/go.mod h1:Jv9NRZ5L2LF87A17GaGlArD+r3JAJdZFvo4XD1cG4Kc=
-k8s.io/mount-utils v0.21.1 h1:uYf6zlKaaoUcPhWn6MElLkWf/f7UQgtkPZteumgwDbA=
-k8s.io/mount-utils v0.21.1/go.mod h1:dwXbIPxKtTjrBEaX1aK/CMEf1KZ8GzMHpe3NEBfdFXI=
 k8s.io/sample-apiserver v0.20.10/go.mod h1:TSEZsVS5JKpV6r1aSVlNlMY17b1Ra2wkuT4LMCRealY=
-k8s.io/sample-apiserver v0.21.1/go.mod h1:haiFU1SSB0+EaaqTpg+CxqOLVwsmjBOJ/CeOk6BTabs=
 k8s.io/system-validators v1.2.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
-k8s.io/system-validators v1.4.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
@@ -1122,17 +938,10 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
-sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY=
-sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0=
-sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
-sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

hack/build-go.sh

@@ -21,7 +21,7 @@ if [ -z "$VERSION" ]; then
 fi
 DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
 COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
-LDFLAGS="-X main.version=${VERSION:-master} -X main.commit=${COMMIT} -X main.date=${DATE}"
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version=${VERSION:-master} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit=${COMMIT} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date=${DATE}"
 export CGO_ENABLED=0
 
 # this if... will be removed when gomodules goes default
@@ -41,6 +41,8 @@ if [ "$GO111MODULE" == "off" ]; then
 	export GOBIN=${PWD}/bin
 	export GOPATH=${PWD}/gopath
 	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
+	go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/config-generation
+	go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/controller/
 else
 	# build with go modules
 	export GO111MODULE=on
@@ -51,4 +53,8 @@ else
 
 	echo "Building plugins"
 	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
+	echo "Building spec generators"
+	go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/config-generation
+	echo "Building multus controller"
+	go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/controller/
 fi

images/Dockerfile

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.1 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -7,9 +7,10 @@ ADD . /usr/src/multus-cni
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM centos:centos7
+FROM python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 WORKDIR /
 
 ADD ./images/entrypoint.sh /

images/Dockerfile.arm32

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.1 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -11,9 +11,10 @@ RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 # build arm container
-FROM arm32v7/centos:7
+FROM arm32v7/python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 
 WORKDIR /
 ADD ./images/entrypoint.sh /

images/Dockerfile.arm64

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.1 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -11,9 +11,10 @@ RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 # build arm64 container
-FROM arm64v8/centos:7
+FROM arm64v8/python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 
 WORKDIR /
 ADD ./images/entrypoint.sh /

images/Dockerfile.ppc64le

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.1 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -11,9 +11,10 @@ RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 # build ppc container
-FROM ppc64le/centos:latest
+FROM ppc64le/python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 
 WORKDIR /
 ADD ./images/entrypoint.sh /

images/Dockerfile.s390x

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -11,9 +11,10 @@ RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 # build s390x container
-FROM s390x/python:3-slim
+FROM s390x/python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 WORKDIR /
 ADD ./images/entrypoint.sh /
 

images/Dockerfile.thick

@@ -0,0 +1,16 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM debian:stable-slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

images/README.md

@@ -5,7 +5,7 @@ This is used for distribution of Multus in a Docker image.
 Typically you'd build this from the root of your Multus clone, as such:
 
 ```
-$ docker build -t dougbtv/multus .
+$ docker build -t dougbtv/multus -f images/Dockerfile .
 ```
 
 ---
@@ -15,7 +15,7 @@ $ docker build -t dougbtv/multus .
 You may wish to deploy Multus as a daemonset, you can do so by starting with the example Daemonset shown here:
 
 ```
-$ kubectl create -f ./images/multus-daemonset.yml
+$ kubectl create -f ./deployments/multus-daemonset.yml
 ```
 
 Note: The likely best practice here is to build your own image given the Dockerfile, and then push it to your preferred registry, and change the `image` fields in the Daemonset YAML to reference that image.
@@ -41,9 +41,7 @@ in lexicographical order in cni-conf-dir).
 ./entrypoint.sh
     -h --help
     --cni-conf-dir=/host/etc/cni/net.d
-    --cni-bin-dir=/host/opt/cni/bin
     --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
-    --multus-bin-file=/usr/src/multus-cni/bin/multus
     --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
 ```
 

images/entrypoint.sh

@@ -36,19 +36,18 @@ SKIP_BINARY_COPY=false
 # Give help text for parameters.
 function usage()
 {
-    echo -e "This is an entrypoint script for Multus CNI to overlay its binary and "
-    echo -e "configuration into locations in a filesystem. The configuration & binary file "
-    echo -e "will be copied to the corresponding configuration directory. When "
-    echo -e "'--multus-conf-file=auto' is used, 00-multus.conf will be automatically "
-    echo -e "generated from the CNI configuration file of the master plugin (the first file "
-    echo -e "in lexicographical order in cni-conf-dir). When '--multus-master-cni-file-name'"
-    echo -e "is used, 00-multus.conf will only be automatically generated from the specific"
-    echo -e "file rather than the first file."
+    echo -e "This is an entrypoint script for Multus CNI to overlay its configuration into"
+    echo -e "locations in a filesystem. The configuration file will be copied to the"
+    echo -e "corresponding configuration directory. When '--multus-conf-file=auto' is used,"
+    echo -e "00-multus.conf will be automatically generated from the CNI configuration file"
+    echo -e "of the master plugin (the first file in lexicographical order in cni-conf-dir)."
+    echo -e "When '--multus-master-cni-file-name' is used, 00-multus.conf will be"
+    echo -e "automatically generated from the specific file rather than the first file."
     echo -e ""
     echo -e "./entrypoint.sh"
     echo -e "\t-h --help"
-    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
     echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
+    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
     echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
     echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
     echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
@@ -84,9 +83,24 @@ function warn()
     log "WARN: {$1}"
 }
 
-if ! type python3 &> /dev/null; then
-	alias python=python3
-fi
+function checkCniVersion {
+      cniversion_python_tmpfile=$(mktemp)
+      cat << EOF > $cniversion_python_tmpfile
+import json, sys
+
+def version(v):
+    return [int(x) for x in v.split(".")]
+
+v_040 = version("0.4.0")
+v_top_level = sys.argv[2]
+with open(sys.argv[1], "r") as f:
+    v_nested = json.load(f)["cniVersion"]
+if version(v_top_level) >= v_040 and version(v_nested) < v_040:
+    msg = "Multus cni version is %s while master plugin cni version is %s"
+    print(msg % (v_top_level, v_nested))
+EOF
+      python3 $cniversion_python_tmpfile $1 $2
+}
 
 # Parse parameters given as arguments to this script.
 while [ "$1" != "" ]; do
@@ -100,6 +114,9 @@ while [ "$1" != "" ]; do
         --cni-version)
             CNI_VERSION=$VALUE
             ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
         --cni-conf-dir)
             CNI_CONF_DIR=$VALUE
             ;;
@@ -109,9 +126,6 @@ while [ "$1" != "" ]; do
         --multus-conf-file)
             MULTUS_CONF_FILE=$VALUE
             ;;
-        --multus-bin-file)
-            MULTUS_BIN_FILE=$VALUE
-            ;;
         --multus-kubeconfig-file-host)
             MULTUS_KUBECONFIG_FILE_HOST=$VALUE
             ;;
@@ -318,7 +332,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
           *)
               error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
               usage
-              exit 1     
+              exit 1
         esac
         LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
       fi
@@ -346,7 +360,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
 
       if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
         MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
+            python3 -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
       else
         MASTER_PLUGIN_NET_NAME="multus-cni-network"
       fi
@@ -368,13 +382,19 @@ else:
 EOF
 
       NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python $capabilities_python_filter_tmpfile)"
+            python3 $capabilities_python_filter_tmpfile)"
       rm $capabilities_python_filter_tmpfile
-      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING" 
+      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING"
 
       MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
       MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
       log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
+      CHECK_CNI_VERSION=$(checkCniVersion $MASTER_PLUGIN_LOCATION $CNI_VERSION)
+      if [ "$CHECK_CNI_VERSION" != "" ] ; then
+        error "$CHECK_CNI_VERSION"
+        exit 1
+      fi
+
       CONF=$(cat <<-EOF
         {
           $CNI_VERSION_STRING
@@ -400,7 +420,7 @@ EOF
       mv $tmpfile $CNI_CONF_DIR/00-multus.conf
       log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
       echo $CONF
-      
+
       # If we're not performing the cleanup on exit, we can safely rename the config file.
       if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
         mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old

pkg/checkpoint/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package checkpoint is the package that contains the libraries that manipulates kubelet's
+// checkpoint API
+package checkpoint

pkg/config/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package config is the package that contains multus cni config related
+// utilities.
+package config

pkg/config/generator.go

@@ -0,0 +1,333 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+)
+
+const (
+	configListCapabilityKey   = "plugins"
+	singleConfigCapabilityKey = "capabilities"
+)
+
+// LogOptionFunc mutates the `LoggingOptions` object
+type LogOptionFunc func(logOptions *LogOptions)
+
+// Option mutates the `conf` object
+type Option func(conf *MultusConf)
+
+// MultusConf holds the multus configuration, and persists it to disk
+type MultusConf struct {
+	BinDir                   string          `json:"binDir,omitempty"`
+	Capabilities             map[string]bool `json:"capabilities,omitempty"`
+	CNIVersion               string          `json:"cniVersion"`
+	Delegates                []interface{}   `json:"delegates"`
+	LogFile                  string          `json:"logFile,omitempty"`
+	LogLevel                 string          `json:"logLevel,omitempty"`
+	LogToStderr              bool            `json:"logToStderr,omitempty"`
+	LogOptions               *LogOptions     `json:"logOptions,omitempty"`
+	Kubeconfig               string          `json:"kubeconfig"`
+	Name                     string          `json:"name"`
+	NamespaceIsolation       bool            `json:"namespaceIsolation,omitempty"`
+	RawNonIsolatedNamespaces string          `json:"globalNamespaces,omitempty"`
+	ReadinessIndicatorFile   string          `json:"readinessindicatorfile,omitempty"`
+	Type                     string          `json:"type"`
+}
+
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// NewMultusConfig creates a basic configuration generator. It can be mutated
+// via the `With...` methods.
+func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
+	multusConfig := &MultusConf{
+		Name:         MultusDefaultNetworkName,
+		CNIVersion:   cniVersion,
+		Type:         pluginName,
+		Capabilities: map[string]bool{},
+		Kubeconfig:   kubeconfig,
+		Delegates:    []interface{}{},
+	}
+
+	err := multusConfig.Mutate(configurationOptions...)
+	return multusConfig, err
+}
+
+// CheckVersionCompatibility checks compatibilty of the
+// top level cni version with the delegate cni version.
+// Since version 0.4.0, CHECK was introduced, which
+// causes incompatibility.
+func CheckVersionCompatibility(mc *MultusConf) error {
+	const versionFmt = "delegate cni version is %s while top level cni version is %s"
+	v040, _ := semver.Make("0.4.0")
+	multusCNIVersion, err := semver.Make(mc.CNIVersion)
+
+	if err != nil {
+		return errors.New("couldn't get top level cni version")
+	}
+
+	if multusCNIVersion.GTE(v040) {
+		for _, delegate := range mc.Delegates {
+			delegatesMap, ok := delegate.(map[string]interface{})
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			delegateVersion, ok := delegatesMap["cniVersion"].(string)
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			v, err := semver.Make(delegateVersion)
+			if err != nil {
+				return err
+			}
+			if v.LT(v040) {
+				return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Generate generates the multus configuration from whatever state is currently
+// held
+func (mc *MultusConf) Generate() (string, error) {
+	data, err := json.Marshal(mc)
+	return string(data), err
+}
+
+// Mutate updates the MultusConf attributes according to the provided
+// configuration `Option`s
+func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
+	for _, configOption := range configurationOptions {
+		configOption(mc)
+	}
+
+	return CheckVersionCompatibility(mc)
+}
+
+// WithNamespaceIsolation mutates the inner state to enable the
+// NamespaceIsolation attribute
+func WithNamespaceIsolation() Option {
+	return func(conf *MultusConf) {
+		conf.NamespaceIsolation = true
+	}
+}
+
+// WithGlobalNamespaces mutates the inner state to set the
+// RawNonIsolatedNamespaces attribute
+func WithGlobalNamespaces(globalNamespaces string) Option {
+	return func(conf *MultusConf) {
+		conf.RawNonIsolatedNamespaces = globalNamespaces
+	}
+}
+
+// WithLogToStdErr mutates the inner state to enable the
+// WithLogToStdErr attribute
+func WithLogToStdErr() Option {
+	return func(conf *MultusConf) {
+		conf.LogToStderr = true
+	}
+}
+
+// WithLogLevel mutates the inner state to set the
+// LogLevel attribute
+func WithLogLevel(logLevel string) Option {
+	return func(conf *MultusConf) {
+		conf.LogLevel = logLevel
+	}
+}
+
+// WithLogFile mutates the inner state to set the
+// logFile attribute
+func WithLogFile(logFile string) Option {
+	return func(conf *MultusConf) {
+		conf.LogFile = logFile
+	}
+}
+
+// WithLogOptions mutates the inner state to set the
+// LogOptions attribute
+func WithLogOptions(logOptions *LogOptions) Option {
+	return func(conf *MultusConf) {
+		conf.LogOptions = logOptions
+	}
+}
+
+// WithReadinessFileIndicator mutates the inner state to set the
+// ReadinessIndicatorFile attribute
+func WithReadinessFileIndicator(path string) Option {
+	return func(conf *MultusConf) {
+		conf.ReadinessIndicatorFile = path
+	}
+}
+
+// WithAdditionalBinaryFileDir mutates the inner state to set the
+// BinDir attribute
+func WithAdditionalBinaryFileDir(directoryPath string) Option {
+	return func(conf *MultusConf) {
+		conf.BinDir = directoryPath
+	}
+}
+
+// WithOverriddenName mutates the inner state to set the
+// Name attribute
+func WithOverriddenName(networkName string) Option {
+	return func(conf *MultusConf) {
+		conf.Name = networkName
+	}
+}
+
+func withCapabilities(cniData interface{}) Option {
+	var enabledCapabilities []string
+	var pluginsList []interface{}
+	cniDataMap, ok := cniData.(map[string]interface{})
+	if ok {
+		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
+			pluginsList = pluginsListEntry.([]interface{})
+		}
+	}
+
+	if len(pluginsList) > 0 {
+		for _, pluginData := range pluginsList {
+			enabledCapabilities = append(
+				enabledCapabilities,
+				extractCapabilities(pluginData)...)
+		}
+	} else {
+		enabledCapabilities = extractCapabilities(cniData)
+	}
+
+	return func(conf *MultusConf) {
+		for _, capability := range enabledCapabilities {
+			conf.Capabilities[capability] = true
+		}
+	}
+}
+
+func withDelegates(primaryCNIConfigData map[string]interface{}) Option {
+	return func(conf *MultusConf) {
+		conf.Delegates = []interface{}{primaryCNIConfigData}
+	}
+}
+
+// MutateLogOptions update the LoggingOptions of the MultusConf according
+// to the provided configuration `loggingOptions`
+func MutateLogOptions(logOption *LogOptions, logOptionFunc ...LogOptionFunc) {
+	for _, loggingOption := range logOptionFunc {
+		loggingOption(logOption)
+	}
+}
+
+// WithLogMaxSize mutates the inner state to set the
+// logMaxSize attribute
+func WithLogMaxSize(maxSize *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxSize = maxSize
+	}
+}
+
+// WithLogMaxAge mutates the inner state to set the
+// logMaxAge attribute
+func WithLogMaxAge(maxAge *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxAge = maxAge
+	}
+}
+
+// WithLogMaxBackups mutates the inner state to set the
+// logMaxBackups attribute
+func WithLogMaxBackups(maxBackups *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxBackups = maxBackups
+	}
+}
+
+// WithLogCompress mutates the inner state to set the
+// logCompress attribute
+func WithLogCompress(compress *bool) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.Compress = compress
+	}
+}
+
+func extractCapabilities(capabilitiesInterface interface{}) []string {
+	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
+	if !ok {
+		return nil
+	}
+	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	var enabledCapabilities []string
+	if len(capabilities) > 0 {
+		for capName, isCapabilityEnabled := range capabilities {
+			if isCapabilityEnabled.(bool) {
+				enabledCapabilities = append(enabledCapabilities, capName)
+			}
+		}
+	}
+	return enabledCapabilities
+}
+
+func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
+	if remainingTries == 0 {
+		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
+	}
+	var cniPluginConfigs []string
+	files, err := ioutil.ReadDir(cniConfigDirPath)
+	if err != nil {
+		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
+	}
+
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "00-multus") {
+			continue
+		}
+		fileExtension := filepath.Ext(file.Name())
+		if fileExtension == ".conf" || fileExtension == ".conflist" {
+			cniPluginConfigs = append(cniPluginConfigs, file.Name())
+		}
+	}
+
+	if len(cniPluginConfigs) == 0 {
+		time.Sleep(time.Second)
+		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
+	}
+	sort.Strings(cniPluginConfigs)
+	return cniPluginConfigs[0], nil
+}

pkg/config/generator_test.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"testing"
+)
+
+const (
+	primaryCNIName = "myCNI"
+	cniVersion     = "0.4.0"
+	kubeconfig     = "/a/b/c/kubeconfig.kubeconfig"
+)
+
+type testCase struct {
+	t                        *testing.T
+	configGenerationFunction func() (string, error)
+}
+
+var primaryCNIConfig = map[string]interface{}{
+	"cniVersion":         "1.0.0",
+	"name":               "ovn-kubernetes",
+	"type":               "ovn-k8s-cni-overlay",
+	"ipam":               "{}",
+	"dns":                "{}",
+	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+	"logLevel":           "5",
+	"logfile-maxsize":    100,
+	"logfile-maxbackups": 5,
+	"logfile-maxage":     5,
+}
+
+func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
+	multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
+	if err != nil {
+		return multusConfig, err
+	}
+	return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{})))
+}
+
+func TestBasicMultusConfig(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig)
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithNamespaceIsolation())
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithReadinessIndicator(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithReadinessFileIndicator("/a/b/u/it-lives"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogLevel("notice"),
+		WithLogToStdErr(),
+		WithLogFile("/u/y/w/log.1"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLogOptionsConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(&LogOptions{
+			MaxAge:     testutils.Int(5),
+			MaxSize:    testutils.Int(100),
+			MaxBackups: testutils.Int(5),
+			Compress:   testutils.Bool(true),
+		}))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5,\"maxSize\":100,\"maxBackups\":5,\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxAge(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxAge(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxSize(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxSize(testutils.Int(100)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxSize\":100},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogBackups(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxBackups(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxBackups\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogCompress(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogCompress(testutils.Bool(true)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithGlobalNamespace(t *testing.T) {
+	const globalNamespace = "come-along-ns"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithGlobalNamespaces(globalNamespace))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
+	const anotherCNIBinDir = "a-dir-somewhere"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithAdditionalBinaryFileDir(anotherCNIBinDir))
+	assertError(t, err, nil)
+	expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`
+{
+    "plugins": [
+        {
+            "capabilities": {
+                "portMappings": true
+            }
+        },
+        {
+            "capabilities": {
+                "tuning": false
+            }
+        }
+    ]
+}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func assertError(t *testing.T, actual error, expected error) {
+	if actual != nil && expected != nil {
+		if actual.Error() != expected.Error() {
+			t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
+		}
+	}
+
+	if actual == nil && expected != nil {
+		t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
+	} else if actual != nil && expected == nil {
+		t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
+	}
+}
+
+func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
+	return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
+}
+
+func TestVersionIncompatibility(t *testing.T) {
+	const delegateCNIVersion = "0.3.0"
+
+	primaryCNIConfigOld := primaryCNIConfig
+	tmpVer := primaryCNIConfig["cniVersion"]
+	primaryCNIConfig["cniVersion"] = delegateCNIVersion
+	_, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfigOld)
+	primaryCNIConfig["cniVersion"] = tmpVer
+
+	assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
+}
+
+func TestMultusConfigWithOverriddenName(t *testing.T) {
+	newNetworkName := "mega-net-2000"
+	multusConfig, _ := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithOverriddenName(newNetworkName))
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
+	return &testCase{
+		t:                        t,
+		configGenerationFunction: configGenerationFunc,
+	}
+}
+
+func (tc testCase) assertResult(expectedResult string) {
+	multusCNIConfig, err := tc.configGenerationFunction()
+	if err != nil {
+		tc.t.Fatalf("error generating multus configuration: %v", err)
+	}
+	if multusCNIConfig != expectedResult {
+		tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
+	}
+}
+
+func documentHelper(pluginInfo string) interface{} {
+	dp, _ := documentCNIData([]byte(pluginInfo))
+	return dp
+}
+
+func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/fsnotify/fsnotify"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+)
+
+// MultusDefaultNetworkName holds the default name of the multus network
+const (
+	multusConfigFileName     = "00-multus.conf"
+	MultusDefaultNetworkName = "multus-cni-network"
+	userRWPermission         = 0600
+)
+
+// Manager monitors the configuration of the primary CNI plugin, and
+// regenerates multus configuration whenever it gets updated.
+type Manager struct {
+	cniConfigData        map[string]interface{}
+	configWatcher        *fsnotify.Watcher
+	multusConfig         *MultusConf
+	multusConfigDir      string
+	multusConfigFilePath string
+	primaryCNIConfigPath string
+}
+
+// NewManager returns a config manager object, configured to persist the
+// configuration to `multusAutoconfigDir`. This constructor will auto-discover
+// the primary CNI for which it will delegate.
+func NewManager(config MultusConf, multusAutoconfigDir string) (*Manager, error) {
+	defaultCNIPluginName, err := primaryCNIPluginName(multusAutoconfigDir)
+	if err != nil {
+		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+		return nil, err
+	}
+	return newManager(config, multusAutoconfigDir, defaultCNIPluginName)
+}
+
+// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
+// configured to persist the configuration to `multusAutoconfigDir`. This
+// constructor will use the primary CNI plugin indicated by the user, via the
+// primaryCNIPluginName variable.
+func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir string, primaryCNIPluginName string) (*Manager, error) {
+	return newManager(config, multusAutoconfigDir, primaryCNIPluginName)
+}
+
+func newManager(config MultusConf, multusConfigDir string, defaultCNIPluginName string) (*Manager, error) {
+	watcher, err := newWatcher(multusConfigDir)
+	if err != nil {
+		return nil, err
+	}
+
+	configManager := &Manager{
+		configWatcher:        watcher,
+		multusConfig:         &config,
+		multusConfigDir:      multusConfigDir,
+		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
+		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+	}
+
+	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
+		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
+	}
+
+	return configManager, nil
+}
+
+func (m *Manager) loadPrimaryCNIConfigFromFile() error {
+	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
+	if err != nil {
+		return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
+	}
+	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
+}
+
+// OverrideNetworkName overrides the name of the multus configuration with the
+// name of the delegated primary CNI.
+func (m *Manager) OverrideNetworkName() error {
+	name, ok := m.cniConfigData["name"]
+	if !ok {
+		return fmt.Errorf("failed to access delegate CNI plugin name")
+	}
+	networkName := name.(string)
+
+	if networkName == "" {
+		return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
+	}
+	return m.multusConfig.Mutate(WithOverriddenName(networkName))
+}
+
+func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
+	cniConfigData := primaryCNIConfigData.(map[string]interface{})
+
+	m.cniConfigData = cniConfigData
+	return m.multusConfig.Mutate(
+		withDelegates(cniConfigData),
+		withCapabilities(cniConfigData))
+}
+
+// GenerateConfig generates a multus configuration from its current state
+func (m Manager) GenerateConfig() (string, error) {
+	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
+		return "", nil
+	}
+	return m.multusConfig.Generate()
+}
+
+// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
+// to by the primaryCNIPluginName attribute, and re-generates the multus
+// configuration whenever the primary CNI config is updated.
+func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
+	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
+
+	for {
+		select {
+		case event := <-m.configWatcher.Events:
+			// we're watching the DIR where the config sits, and the event
+			// does not concern the primary CNI config. Skip it.
+			if event.Name != m.primaryCNIConfigPath {
+				logging.Debugf("skipping un-related event %v", event)
+				continue
+			}
+
+			if !shouldRegenerateConfig(event) {
+				continue
+			}
+
+			updatedConfig, err := m.GenerateConfig()
+			if err != nil {
+				_ = logging.Errorf("failed to regenerate the multus configuration: %v", err)
+			}
+
+			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
+			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+			}
+			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+				_ = logging.Errorf("failed to reload the updated config: %v", err)
+			}
+
+		case err := <-m.configWatcher.Errors:
+			if err == nil {
+				continue
+			}
+			logging.Errorf("CNI monitoring error %v", err)
+
+		case <-shutDown:
+			logging.Verbosef("Stopped monitoring, closing channel ...")
+			_ = m.configWatcher.Close()
+			done <- struct{}{}
+			return nil
+		}
+	}
+}
+
+// PersistMultusConfig persists the provided configuration to the disc, with
+// Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
+func (m Manager) PersistMultusConfig(config string) error {
+	return ioutil.WriteFile(m.multusConfigFilePath, []byte(config), userRWPermission)
+}
+
+func primaryCNIPluginName(multusAutoconfigDir string) (string, error) {
+	masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
+	if err != nil {
+		return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
+	}
+	return masterCniConfigFileName, nil
+}
+
+func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
+	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
+}
+
+func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
+	}
+	defer func() {
+		// Close watcher on error
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+
+	if err = watcher.Add(cniConfigDir); err != nil {
+		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+	}
+
+	return watcher, nil
+}
+
+func shouldRegenerateConfig(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Write == fsnotify.Write ||
+		event.Op&fsnotify.Create == fsnotify.Create
+}
+
+func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
+	masterCNIConfigData, err := ioutil.ReadFile(masterCNIPluginPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
+	}
+
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall primary CNI config: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const suiteName = "Configuration Manager"
+
+func TestMultusConfigurationManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, suiteName)
+}
+
+var _ = Describe(suiteName, func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var configManager *Manager
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	BeforeEach(func() {
+		var err error
+		multusConfigDir, err = ioutil.TempDir("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+	})
+
+	BeforeEach(func() {
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), userRWPermission)).To(Succeed())
+
+		multusConf, _ := NewMultusConfig(
+			primaryCNIName,
+			cniVersion,
+			kubeconfig)
+		var err error
+		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
+		expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(config).To(Equal(expectedResult))
+	})
+
+	Context("Updates to the delegate CNI configuration", func() {
+		var (
+			doneChannel chan struct{}
+			stopChannel chan struct{}
+		)
+
+		BeforeEach(func() {
+			doneChannel = make(chan struct{})
+			stopChannel = make(chan struct{})
+			go func() {
+				Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
+			}()
+		})
+
+		AfterEach(func() {
+			go func() { stopChannel <- struct{}{} }()
+			Eventually(<-doneChannel).Should(Equal(struct{}{}))
+			close(doneChannel)
+			close(stopChannel)
+		})
+
+		It("Trigger the re-generation of the Multus CNI configuration", func() {
+			newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
+			Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), userRWPermission)).To(Succeed())
+
+			multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
+			Eventually(func() (string, error) {
+				multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
+				return string(multusCniData), err
+			}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
+		})
+	})
+
+	When("the user requests the name of the multus configuration to be overridden", func() {
+		BeforeEach(func() {
+			Expect(configManager.OverrideNetworkName()).To(Succeed())
+		})
+
+		It("Overrides the name of the multus configuration when requested", func() {
+			expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
+			config, err := configManager.GenerateConfig()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(expectedResult))
+		})
+	})
+})
+
+func multusConfigFromDelegate(delegateConfig string) string {
+	return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
+}

pkg/k8sclient/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package k8sclient is the package that contains the Kubernetes client libraries.
+package k8sclient

pkg/k8sclient/k8sclient.go

@@ -92,7 +92,7 @@ func (c *ClientInfo) Eventf(object runtime.Object, eventtype, reason, messageFmt
 	}
 }
 
-func (e *NoK8sNetworkError) Error() string { return string(e.message) }
+func (e *NoK8sNetworkError) Error() string { return e.message }
 
 // SetNetworkStatus sets network status into Pod annotation
 func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
@@ -114,11 +114,16 @@ func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []ne
 
 	podName := string(k8sArgs.K8S_POD_NAME)
 	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podUID := string(k8sArgs.K8S_POD_UID)
 	pod, err := client.GetPod(podNamespace, podName)
 	if err != nil {
 		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", podName, err)
 	}
 
+	if podUID != "" && string(pod.UID) != podUID && !IsStaticPod(pod) {
+		return logging.Errorf("SetNetworkStatus: expected pod %s/%s UID %q but got %q from Kube API", podNamespace, podName, podUID, pod.UID)
+	}
+
 	if netStatus != nil {
 		err = netutils.SetNetworkStatus(client.Client, pod, netStatus)
 		if err != nil {
@@ -361,10 +366,13 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 		}
 
 		if isGatewayConfigured == true {
-			types.CheckGatewayConfig(conf.Delegates)
+			err = types.CheckGatewayConfig(conf.Delegates)
+			if err != nil {
+				return 0, nil, err
+			}
 		}
 
-		return len(delegates), clientInfo, nil
+		return len(delegates), clientInfo, err
 	}
 
 	if _, ok := err.(*NoK8sNetworkError); ok {
@@ -611,3 +619,16 @@ func tryLoadK8sPodDefaultNetwork(kubeClient *ClientInfo, pod *v1.Pod, conf *type
 
 	return delegate, nil
 }
+
+// ConfigSourceAnnotationKey specifies kubernetes annotation, defined in k8s.io/kubernetes/pkg/kubelet/types
+const ConfigSourceAnnotationKey = "kubernetes.io/config.source"
+
+// IsStaticPod returns true if the pod is static pod.
+func IsStaticPod(pod *v1.Pod) bool {
+	if pod.Annotations != nil {
+		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok == true {
+			return source != "api"
+		}
+	}
+	return false
+}

pkg/k8sclient/k8sclient_test.go

@@ -55,6 +55,9 @@ var _ = Describe("k8sclient operations", func() {
 	var tmpDir string
 	var err error
 	var genericConf string
+	var args *skel.CmdArgs
+
+	const fakePodName string = "testPod"
 
 	BeforeEach(func() {
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
@@ -69,6 +72,11 @@ var _ = Describe("k8sclient operations", func() {
 			}],
 			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
 		}`
+
+		args = &skel.CmdArgs{
+			// Values come from NewFakePod()
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePodName, "test", "testUID"),
+		}
 	})
 
 	AfterEach(func() {
@@ -77,7 +85,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using simple format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -94,10 +102,6 @@ var _ = Describe("k8sclient operations", func() {
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -130,15 +134,12 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the network does not exist", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 		net3 := `{
 	"name": "net3",
 	"type": "mynet3",
 	"cniVersion": "0.2.0"
 }`
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -160,7 +161,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using JSON format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", `[
+		fakePod := testutils.NewFakePod(fakePodName, `[
 {"name":"net1"},
 {
   "name":"net2",
@@ -172,9 +173,6 @@ var _ = Describe("k8sclient operations", func() {
   "namespace":"other-ns"
 }
 ]`, "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -218,10 +216,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the JSON format annotation is invalid", func() {
-		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "[adsfasdfasdfasf]", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -236,7 +231,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("can set the default-gateway on an additional interface", func() {
-		fakePod := testutils.NewFakePod("testpod", `[
+		fakePod := testutils.NewFakePod(fakePodName, `[
 {"name":"net1"},
 {
   "name":"net2",
@@ -247,9 +242,6 @@ var _ = Describe("k8sclient operations", func() {
   "namespace":"other-ns"
 }
 ]`, "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -294,10 +286,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using on-disk config files", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -338,10 +327,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("injects network name into minimal thick plugin CNI config", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -365,10 +351,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when on-disk config file is not valid", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -399,7 +382,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from CRD", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -409,10 +392,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -431,7 +410,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves default networks from CRD", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -442,10 +421,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -469,7 +444,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("ignore default networks from CRD in case of kube-system namespace", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		// overwrite namespace
 		fakePod.ObjectMeta.Namespace = "kube-system"
 		conf := `{
@@ -482,10 +457,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -507,7 +478,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from file", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -518,10 +489,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf.ConfDir = tmpDir
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -546,7 +513,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from path", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -556,10 +523,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -582,7 +545,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("Error in case of CRD not found", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -592,10 +555,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -608,7 +567,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("overwrite cluster network when Pod annotation is set", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -619,10 +578,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -650,7 +605,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with bad confdir", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -661,10 +616,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -686,7 +637,7 @@ var _ = Describe("k8sclient operations", func() {
 
 	It("overwrite multus config when Pod annotation is set", func() {
 
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -701,10 +652,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -723,7 +670,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with no kubeclient and invalid kubeconfig", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -738,10 +685,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -757,7 +700,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with no kubeclient and no kubeconfig", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -772,10 +715,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -821,7 +760,7 @@ users:
     client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMUxyZmUxaXNQaTNDYng4eHRDWG1QTlE2L0xXYVVEa1ZRNVBkdGtQK3dVYmdJUFJ6CmNtRzNEcnUya0kvamsxMGQzaUgzOVZlZDlHVjczbnZyVm5Id0pzZEF5QzVmYlRTQVBUVVNJOW90MVFGbENRUGcKQ2JLbjBRbmxUeUhXMmNnTis2Q3BWK0U4dFpWajJ2cDZoTDlmR0s2bUJCL2I4VFN3RmYxRmtnWC9hTGN1emZDaApmVFNDOWlRTk15cEFjamZadEkzWiszbHVzSVB6TVpBNTZPZXNodzlnRkJHUkw3RzZHWmZKcG9OaGVxTDlmandUClRkNURlbVZXcUxUR05ZNWhhV3hicy9VbThuM0QvdlV1QkVPYTYxQ2cvcGljUkU3Um14SmFJRWJiQlNTV1dkSDMKWDlrem5RdHJrUXloL28xZnpSV1pxeWFCY3hxR1FWN0JSSzFtSndJREFRQUJBb0lCQUJ0bjA4QzFSTU5oNjhtYgpFREV3TE1BcmEwb0JMMWNrYzN2WVFkam9XNXFVd2UwYzhQNk1YaVAweE9sTTBEbTg1a3NtdnlZSldwMFFzZXVRCnRWbldwZVNwQ015QlJPUHh2bytrRmFrdXczYk1qaktpSUN1L3EyVC96RjNzY3h4dGJIZTlVL094WGJ2YStobE0KNlpuT2ViYlpVU1A0NHNIcFVzSVNkZk1BK00ySmg1UFJibGZWaUFEY1hxNFR5RU1JaStzRkhOcFIrdmdWZzRFawp4RmFVaS83V0E2YUxWVzBUTzREdjMwbTJ0TVczWXN1bk1LTU0xOTNyUEZrU0dEdFpheWV2Z0JDeURXaFhOTEo2Clh1cTNxSUg4bFE2bzRBUjMvcDc1ZW9hOCtrVzVmT3o2UWF3WnpPYlBENkRCQlVOYVM1YklXaVV1dmx5L0JlM20KZnlxK3NRRUNnWUVBMW84R3l6ODk2bFhwdU1yVXVsb2orbGp5U0FaNkpLOCsyaFMvQnpyREx6NlpvY3FzKzg3awpVUkwzKy9LL1pja2pIMVVDeXROVVZ6Q3RKaG4zZmdLS2dpQWhsU0pNRzhqc05sbEkydFZSazNZZ1RCcUg2bXZxCit3citsTUxoUDZxbWFObUx3QXljY2lEanpMdXlRdjhVOFhKazJOdVFsQlFwbkt2eWJIRGdxSUVDZ1lFQS9kRnMKazNlYmRNNFAxV2psYXJoRTV5blpuRmdQbDg1L2Vudk4rQ1oxcStlMGxYendaUGswdWdJUWozYyt2UEpLWlh0OApLWk1HQjM0N2VLNlFIL3J1a2xRWXlLOStHeUV1YnRJQUZ2NWFrYXZxV1haR1p5ZC9QdDR1V09adXMrd3BnSG00CkxFY0lzZElsYkpFY2RJTzJyb3FaY0VNY3FEbGtXcTdwQWxqU2VxY0NnWUJYdUQ0RTFxUlByRFJVRXNrS0wxUksKUkJjNkR6dmN4N0VncEI2OXErNms0Q2tibHF0R2YvMmtqK2JISVNYVFRYcUlrczhEY1ljbjVvVEQ4UlhZZE4xLworZmNBNi9iRjNVMkZvdGRBY0xwYldZNDJ6eG9HWTN5OGluQXZEY1hkcTcxQlhML2dFc2ZiZVVycEowdm9URFdaCnlUVWwzQTZ1RzlndmI3VTdWS0xsQVFLQmdBTmNscmVOU2YzT0ROK2l1QWNsMGFQT0poZXdBdVRiMDB4bi8xNWUKQkFqMjFLbDJNaWprTkJLU25HMktBc2ExM3M1aFNFKzBwc3ZLbkRjSStOZXpseDFSQjlNQW9BYno5WTE2TW80YgphRSt0bXpqOEhBcVp0MUc1MTV0TjBnR0lDelNzYUFnT0dNdGlJU1RDOTBHRHpST2F1bFdHVGdiY1c3dm52U1pPCnp0clpBb0dBWmtIRWV5em16Z2cxR3dtTzN3bmljSHRMR1BQRFhiSW53NTdsdkIrY3lyd0FrVEs1MlFScTM0VkMKRDhnQWFwMTU2OWlWUER3YlgrNkpBQk1WQ2tNUmdxMjdHanUzN0pVY2Fib2g1YzJQeTBYNUlhUG8rek1hWHgvQwpqbjUvUW5YandjU1MrRU5hL1lXVWcxWEVjQjJYdEM0UExCdGUycitrUTVLbFNOREcxSTQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==`
 
 		kubeletconf.Write([]byte(kubeletconfDef))
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -836,10 +775,6 @@ users:
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -854,7 +789,7 @@ users:
 	})
 
 	It("Errors when namespace isolation is violated", func() {
-		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -875,10 +810,6 @@ users:
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -902,7 +833,7 @@ users:
 	})
 
 	It("Properly allows a specified namespace reference when namespace isolation is enabled", func() {
-		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -924,10 +855,6 @@ users:
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -958,7 +885,7 @@ users:
 
 	Context("getDefaultNetDelegateCRD", func() {
 		It("fails when netConf contains bad confDir", func() {
-			fakePod := testutils.NewFakePod("testpod", "", "net1")
+			fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 			conf := `{
 				"name":"node-cni-network",
 				"type":"multus",
@@ -969,10 +896,6 @@ users:
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -992,9 +915,9 @@ users:
 
 	Context("GetK8sArgs", func() {
 		It("fails when provided with bad format", func() {
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME:%s;K8S_POD_NAMESPACE:%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME:%s;K8S_POD_NAMESPACE:%s;K8S_POD_UID:%s", fakePod.Name, fakePod.Namespace, fakePod.UID),
 			}
 			// using colon instead of equals sign makes an invalid CmdArgs
 
@@ -1005,7 +928,7 @@ users:
 
 	Context("getKubernetesDelegate", func() {
 		It("failed to get a ResourceClient instance", func() {
-			fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+			fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 			net1 := `{
 		"name": "net1",
 		"type": "mynet",
@@ -1021,9 +944,6 @@ users:
 		"type": "mynet3",
 		"cniVersion": "0.2.0"
 	}`
-			// args := &skel.CmdArgs{
-			// 	Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			// }
 
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
@@ -1051,10 +971,7 @@ users:
 
 	Context("parsePodNetworkObjectName", func() {
 		It("fails to get podnetwork given bad annotation values", func() {
-			fakePod := testutils.NewFakePod("testpod", "net1", "")
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
+			fakePod := testutils.NewFakePod(fakePodName, "net1", "")
 
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
@@ -1087,7 +1004,7 @@ users:
 
 	Context("setPodNetworkAnnotation", func() {
 		It("Sets pod network annotations without error", func() {
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			net1 := `{
 		"name": "net1",
@@ -1095,10 +1012,6 @@ users:
 		"cniVersion": "0.2.0"
 	}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1172,10 +1085,6 @@ users:
 		"cniVersion": "0.2.0"
 	}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1183,6 +1092,9 @@ users:
 			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
 			Expect(err).NotTo(HaveOccurred())
 
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePod.Name, fakePod.Namespace, "blahblah"),
+			}
 			k8sArgs, err := GetK8sArgs(args)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1209,7 +1121,7 @@ users:
 
 		// TODO Still figuring this next one out. deals with exponentialBackoff
 		// It("Fails to set pod network annotations without error", func() {
-		// 	fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		// 	fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 		// 	net1 := `{
 		// 	"name": "net1",
@@ -1217,10 +1129,6 @@ users:
 		// 	"cniVersion": "0.2.0"
 		// }`
 
-		// 	args := &skel.CmdArgs{
-		// 		Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		// 	}
-
 		//	clientInfo := NewFakeClientInfo()
 		//	_, err := clientInfo.AddPod(fakePod)
 		//	Expect(err).NotTo(HaveOccurred())
@@ -1240,7 +1148,7 @@ users:
 	})
 
 	Context("SetNetworkStatus", func() {
-		It("Sets network status without error", func() {
+		It("Sets network status without error when pod UIDs match", func() {
 			result := &types020.Result{
 				CNIVersion: "0.2.0",
 				IP4: &types020.IPConfig{
@@ -1271,7 +1179,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1282,10 +1190,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1299,6 +1203,123 @@ users:
 			Expect(err).NotTo(HaveOccurred())
 		})
 
+		It("Sets pod network annotations without error when runtime does not provide a pod UID", func() {
+			result := &types020.Result{
+				CNIVersion: "0.2.0",
+				IP4: &types020.IPConfig{
+					IP: *testutils.EnsureCIDR("1.1.1.2/24"),
+				},
+			}
+
+			conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "weave-net"
+			}],
+		  "runtimeConfig": {
+			  "portMappings": [
+				{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			  ]
+			}
+		}`
+
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
+			Expect(err).NotTo(HaveOccurred())
+
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			Expect(err).NotTo(HaveOccurred())
+
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+
+			netConf, err := types.LoadNetConf([]byte(conf))
+			Expect(err).NotTo(HaveOccurred())
+
+			net1 := `{
+			"name": "net1",
+			"type": "mynet",
+			"cniVersion": "0.2.0"
+		}`
+
+			clientInfo := NewFakeClientInfo()
+			_, err = clientInfo.AddPod(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
+			Expect(err).NotTo(HaveOccurred())
+
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.Name, fakePod.Namespace),
+			}
+			k8sArgs, err := GetK8sArgs(args)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = SetNetworkStatus(clientInfo, k8sArgs, netstatus, netConf)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("Fails to set pod network annotations when pod UIDs don't match", func() {
+			result := &types020.Result{
+				CNIVersion: "0.2.0",
+				IP4: &types020.IPConfig{
+					IP: *testutils.EnsureCIDR("1.1.1.2/24"),
+				},
+			}
+
+			conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "weave-net"
+			}],
+		  "runtimeConfig": {
+			  "portMappings": [
+				{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			  ]
+			}
+		}`
+
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
+			Expect(err).NotTo(HaveOccurred())
+
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			Expect(err).NotTo(HaveOccurred())
+
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+
+			netConf, err := types.LoadNetConf([]byte(conf))
+			Expect(err).NotTo(HaveOccurred())
+
+			net1 := `{
+			"name": "net1",
+			"type": "mynet",
+			"cniVersion": "0.2.0"
+		}`
+
+			clientInfo := NewFakeClientInfo()
+			_, err = clientInfo.AddPod(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
+			Expect(err).NotTo(HaveOccurred())
+
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePod.Name, fakePod.Namespace, "foobar"),
+			}
+			k8sArgs, err := GetK8sArgs(args)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = SetNetworkStatus(clientInfo, k8sArgs, netstatus, netConf)
+			Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("expected pod %s/%s UID %q but got %q from Kube API",
+				fakePod.Namespace, fakePod.Name, string(k8sArgs.K8S_POD_UID), fakePod.UID)))
+		})
+
 		It("Sets network status with kubeclient built from kubeconfig and attempts to connect", func() {
 			kubeletconf, err := os.Create("/etc/kubernetes/kubelet.conf")
 			kubeletconfDef := `apiVersion: v1
@@ -1353,7 +1374,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1364,10 +1385,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1413,7 +1430,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1424,10 +1441,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1472,7 +1485,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1483,10 +1496,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())

pkg/kubeletclient/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package kubeletclient is the package that contains the kubelet's libraries that
+// controls podresource API in kubelet
+package kubeletclient

pkg/kubeletclient/kubeletclient_test.go

@@ -71,6 +71,7 @@ func TestKubeletclient(t *testing.T) {
 }
 
 var testKubeletSocket string
+
 func setUp() error {
 	tempSocketDir, err := ioutil.TempDir("", "kubelet-resource-client")
 	if err != nil {

pkg/logging/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package logging is the package that contains logging library.
+package logging

pkg/logging/logging.go

@@ -41,9 +41,42 @@ const (
 var loggingStderr bool
 var loggingW io.Writer
 var loggingLevel Level
+var logger *lumberjack.Logger
 
 const defaultTimestampFormat = time.RFC3339
 
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// SetLogOptions set the LoggingOptions of NetConf
+func SetLogOptions(options *LogOptions) {
+	// give some default value
+	logger.MaxSize = 100
+	logger.MaxAge = 5
+	logger.MaxBackups = 5
+	logger.Compress = true
+	if options != nil {
+		if options.MaxAge != nil {
+			logger.MaxAge = *options.MaxAge
+		}
+		if options.MaxSize != nil {
+			logger.MaxSize = *options.MaxSize
+		}
+		if options.MaxBackups != nil {
+			logger.MaxBackups = *options.MaxBackups
+		}
+		if options.Compress != nil {
+			logger.Compress = *options.Compress
+		}
+	}
+	loggingW = logger
+}
+
 func (l Level) String() string {
 	switch l {
 	case PanicLevel:
@@ -141,13 +174,8 @@ func SetLogFile(filename string) {
 		return
 	}
 
-	loggingW = &lumberjack.Logger{
-		Filename:   filename,
-		MaxSize:    100, // megabytes
-		MaxBackups: 5,
-		MaxAge:     5, // days
-		Compress:   true,
-	}
+	logger.Filename = filename
+	loggingW = logger
 
 }
 
@@ -155,4 +183,5 @@ func init() {
 	loggingStderr = true
 	loggingW = nil
 	loggingLevel = PanicLevel
+	logger = &lumberjack.Logger{}
 }

pkg/logging/logging_test.go

@@ -15,6 +15,8 @@
 package logging
 
 import (
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"gopkg.in/natefinch/lumberjack.v2"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -79,4 +81,54 @@ var _ = Describe("logging operations", func() {
 		currentLevel := loggingLevel
 		Expect(currentLevel).To(Equal(GetLoggingLevel()))
 	})
+
+	It("Check user settings logOptions for logging", func() {
+		SetLogFile("/var/log/multus.log")
+		expectLogger := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     1,
+			MaxSize:    10,
+			MaxBackups: 1,
+			Compress:   true,
+		}
+		logOptions := &LogOptions{
+			MaxAge:     testutils.Int(1),
+			MaxSize:    testutils.Int(10),
+			MaxBackups: testutils.Int(1),
+			Compress:   testutils.Bool(true),
+		}
+		SetLogOptions(logOptions)
+		Expect(expectLogger).To(Equal(logger))
+	})
+
+	It("Check user settings logOptions and missing some options", func() {
+		SetLogFile("/var/log/multus.log")
+		expectLogger := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     5,
+			MaxSize:    100,
+			MaxBackups: 1,
+			Compress:   true,
+		}
+		logOptions := &LogOptions{
+			MaxBackups: testutils.Int(1),
+			Compress:   testutils.Bool(true),
+		}
+		SetLogOptions(logOptions)
+		Expect(expectLogger).To(Equal(logger))
+	})
+
+	It("Check user don't settings logOptions for logging", func() {
+		SetLogFile("/var/log/multus.log")
+		logger1 := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     5,
+			MaxSize:    100,
+			MaxBackups: 5,
+			Compress:   true,
+		}
+		SetLogOptions(nil)
+		Expect(logger1).To(Equal(logger))
+	})
+
 })

pkg/multus/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package multus is the package that contains main multus function, which
+// manipulates CNI request for delegate plugins.
+package multus

pkg/multus/multus.go

@@ -34,14 +34,15 @@ import (
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/vishvananda/netlink"
-	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/netutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	k8snet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/netutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 )
 
 const (
@@ -60,7 +61,7 @@ var (
 	pollTimeout  = 45 * time.Second
 )
 
-//PrintVersionString ...
+// PrintVersionString ...
 func PrintVersionString() string {
 	return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s",
 		version, commit, date)
@@ -199,14 +200,14 @@ func confCheck(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.N
 
 	err = cniNet.CheckNetwork(context.Background(), conf, rt)
 	if err != nil {
-		return logging.Errorf("error in getting result from DelNetwork: %v", err)
+		return logging.Errorf("error in getting result from CheckNetwork: %v", err)
 	}
 
 	return err
 }
 
 func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
-	logging.Debugf("conflistDel: %v, %s", rt, string(rawNetconf))
+	logging.Debugf("confDel: %v, %s", rt, string(rawNetconf))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
 	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
@@ -285,23 +286,15 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *t
 	return err
 }
 
-func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf, cniArgs string) (cnitypes.Result, error) {
-	logging.Debugf("delegateAdd: %v, %s, %v, %v", exec, ifName, delegate, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return nil, logging.Errorf("delegateAdd: error setting environment variable CNI_IFNAME")
-	}
+func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
+	logging.Debugf("delegateAdd: %v, %v, %v", exec, delegate, rt)
 
-	if err := validateIfName(os.Getenv("CNI_NETNS"), ifName); err != nil {
-		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, ifName, err)
+	if err := validateIfName(rt.NetNS, rt.IfName); err != nil {
+		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, rt.IfName, err)
 	}
 
 	// Deprecated in ver 3.5.
 	if delegate.MacRequest != "" || delegate.IPRequest != nil {
-		if cniArgs != "" {
-			cniArgs = fmt.Sprintf("%s;IgnoreUnknown=true", cniArgs)
-		} else {
-			cniArgs = "IgnoreUnknown=true"
-		}
 		if delegate.MacRequest != "" {
 			// validate Mac address
 			_, err := net.ParseMAC(delegate.MacRequest)
@@ -309,8 +302,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 				return nil, logging.Errorf("delegateAdd: failed to parse mac address %q", delegate.MacRequest)
 			}
 
-			cniArgs = fmt.Sprintf("%s;MAC=%s", cniArgs, delegate.MacRequest)
-			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, ifName)
+			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"MAC", delegate.MacRequest})
 		}
 
@@ -328,8 +320,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 			}
 
 			ips := strings.Join(delegate.IPRequest, ",")
-			cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, ips)
-			logging.Debugf("delegateAdd: set IP address %q to %q", ips, ifName)
+			logging.Debugf("delegateAdd: set IP address %q to %q", ips, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"IP", ips})
 		}
 	}
@@ -376,11 +367,14 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 	}
 
 	if pod != nil {
-		// send kubernetes events
-		if delegate.Name != "" {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
-		} else {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+		// check Interfaces and IPs because some CNI plugin just return empty result
+		if res.Interfaces != nil || res.IPs != nil {
+			// send kubernetes events
+			if delegate.Name != "" {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
+			} else {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+			}
 		}
 	} else {
 		// for further debug https://github.com/k8snetworkplumbingwg/multus-cni/issues/481
@@ -390,11 +384,8 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 	return result, nil
 }
 
-func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateCheck: %v, %s, %v, %v", exec, ifName, delegateConf, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateCheck: error setting environment variable CNI_IFNAME")
-	}
+func delegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("delegateCheck: %v, %v, %v", exec, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var cniConfName string
@@ -422,11 +413,8 @@ func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.Delegate
 	return err
 }
 
-func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateDel: %v, %v, %s, %v, %v", exec, pod, ifName, delegateConf, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateDel: error setting environment variable CNI_IFNAME")
-	}
+func delegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("delegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var confName string
@@ -463,16 +451,13 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 // with each of the delegates' configuration
 func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, multusNetconf *types.NetConf) error {
 	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v", exec, pod, args, k8sArgs, delegates, lastIdx, netRt)
-	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return logging.Errorf("delPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
-	}
 
 	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
 		ifName := getIfname(delegates[idx], args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
-		if err := delegateDel(exec, pod, ifName, delegates[idx], rt, multusNetconf); err != nil {
+		if err := delegateDel(exec, pod, delegates[idx], rt, multusNetconf); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
 		if cniDeviceInfoPath != "" {
@@ -496,7 +481,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error {
 	prefix := "Multus: "
 	if k8sArgs != nil {
-		prefix += fmt.Sprintf("[%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME)
+		prefix += fmt.Sprintf("[%s/%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_UID)
 	}
 	return logging.Errorf(prefix+format, args...)
 }
@@ -504,7 +489,7 @@ func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error {
 func cmdPluginErr(k8sArgs *types.K8sArgs, confName string, format string, args ...interface{}) error {
 	msg := ""
 	if k8sArgs != nil {
-		msg += fmt.Sprintf("[%s/%s:%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, confName)
+		msg += fmt.Sprintf("[%s/%s/%s:%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_UID, confName)
 	}
 	return logging.Errorf(msg+format, args...)
 }
@@ -521,7 +506,52 @@ func isCriticalRequestRetriable(err error) bool {
 	return false
 }
 
-//CmdAdd ...
+func getPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
+	if kubeClient == nil {
+		return nil, nil
+	}
+
+	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	podUID := string(k8sArgs.K8S_POD_UID)
+
+	pod, err := kubeClient.GetPod(podNamespace, podName)
+	if err != nil {
+		// in case of a retriable error, retry 10 times with 0.25 sec interval
+		if isCriticalRequestRetriable(err) {
+			waitErr := wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
+				pod, err = kubeClient.GetPod(podNamespace, podName)
+				return pod != nil, err
+			})
+			// retry failed, then return error with retry out
+			if waitErr != nil {
+				return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
+			}
+		} else if warnOnly && errors.IsNotFound(err) {
+			// If not found, proceed to remove interface with cache
+			return nil, nil
+		} else {
+			// Other case, return error
+			return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
+		}
+	}
+
+	// In case of static pod, UID through kube api is different because of mirror pod, hence it is expected.
+	if podUID != "" && string(pod.UID) != podUID && !k8s.IsStaticPod(pod) {
+		msg := fmt.Sprintf("expected pod UID %q but got %q from Kube API", podUID, pod.UID)
+		if warnOnly {
+			// On CNI DEL we just operate on the cache when these mismatch, we don't error out.
+			// For example: stateful sets namespace/name can remain the same while podUID changes.
+			logging.Verbosef("warning: %s", msg)
+			return nil, nil
+		}
+		return nil, cmdErr(k8sArgs, msg)
+	}
+
+	return pod, nil
+}
+
+// CmdAdd ...
 func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) {
 	n, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdAdd: %v, %v, %v", args, exec, kubeClient)
@@ -549,26 +579,9 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 	}
 
-	pod := (*v1.Pod)(nil)
-	if kubeClient != nil {
-		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-		if err != nil {
-			var waitErr error
-			// in case of a retriable error, retry 10 times with 0.25 sec interval
-			if isCriticalRequestRetriable(err) {
-				waitErr = wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
-					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-					return pod != nil, err
-				})
-				// retry failed, then return error with retry out
-				if waitErr != nil {
-					return nil, cmdErr(k8sArgs, "error getting pod with error: %v", err)
-				}
-			} else {
-				// Other case, return error
-				return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
-			}
-		}
+	pod, err := getPod(kubeClient, k8sArgs, false)
+	if err != nil {
+		return nil, err
 	}
 
 	// resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation.
@@ -596,7 +609,6 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 	var result, tmpResult cnitypes.Result
 	var netStatus []nettypes.NetworkStatus
-	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
@@ -609,57 +621,95 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			}
 		}
 
-		tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n, cniArgs)
+		// We collect the delegate netName for the cachefile name as well as following errors
+		netName := delegate.Conf.Name
+		if netName == "" {
+			netName = delegate.ConfList.Name
+		}
+		tmpResult, err = delegateAdd(exec, kubeClient, pod, delegate, rt, n)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
-			netName := delegate.Conf.Name
-			if netName == "" {
-				netName = delegate.ConfList.Name
-			}
 			// Ignore errors; DEL must be idempotent anyway
 			_ = delPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n)
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
-		// Remove gateway from routing table if the gateway is not used
-		deletegateway := false
-		adddefaultgateway := false
-		if delegate.IsFilterGateway {
-			deletegateway = true
-			logging.Debugf("Marked interface %v for gateway deletion", ifName)
-		} else {
-			// Otherwise, determine if this interface now gets our default route.
-			// According to
-			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
-			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
-			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && delegate.GatewayRequest[0] != nil {
-				deletegateway = true
-				adddefaultgateway = true
-				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
-			}
-		}
-
-		if deletegateway {
-			tmpResult, err = netutils.DeleteDefaultGW(args, ifName, &tmpResult)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
-			}
-		}
-
-		// Here we'll set the default gateway
-		if adddefaultgateway {
-			tmpResult, err = netutils.SetDefaultGW(args, ifName, delegate.GatewayRequest, &tmpResult)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
-			}
-		}
-
 		// Master plugin result is always used if present
 		if delegate.MasterPlugin || result == nil {
 			result = tmpResult
 		}
 
+		res, err := cnicurrent.NewResultFromResult(tmpResult)
+		if err != nil {
+			logging.Errorf("CmdAdd: failed to read result: %v, but proceed", err)
+		}
+
+		// check Interfaces and IPs because some CNI plugin does not create any interface
+		// and just returns empty result
+		// !bang
+		if res != nil && (res.Interfaces != nil || res.IPs != nil) {
+
+			// Remove gateway from routing table if the gateway is not used
+			deleteV4gateway := false
+			deleteV6gateway := false
+			adddefaultgateway := false
+			if delegate.IsFilterV4Gateway {
+				deleteV4gateway = true
+				logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV4gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			if delegate.IsFilterV6Gateway {
+				deleteV6gateway = true
+				logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV6gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			// Remove gateway if `default-route` network selection is specified
+			if deleteV4gateway || deleteV6gateway {
+				err = netutils.DeleteDefaultGW(args, ifName)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
+				}
+				err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
+				}
+			}
+
+			// Here we'll set the default gateway which specified in `default-route` network selection
+			if adddefaultgateway {
+				err = netutils.SetDefaultGW(args, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
+				}
+				err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
+				}
+			}
+		}
+
 		// Read devInfo from CNIDeviceInfoFile if it exists so
 		// it can be copied to the NetworkStatus.
 		devinfo, err := getDelegateDeviceInfo(delegate, rt)
@@ -701,7 +751,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	return result, nil
 }
 
-//CmdCheck ...
+// CmdCheck ...
 func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdCheck: %v, %v, %v", args, exec, kubeClient)
@@ -718,7 +768,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 		ifName := getIfname(delegate, args.IfName, idx)
 
 		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
-		err = delegateCheck(exec, ifName, delegate, rt, in)
+		err = delegateCheck(exec, delegate, rt, in)
 		if err != nil {
 			return err
 		}
@@ -727,7 +777,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 	return nil
 }
 
-//CmdDel ...
+// CmdDel ...
 func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdDel: %v, %v, %v", args, exec, kubeClient)
@@ -735,18 +785,18 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return err
 	}
 
-	netnsfound := true
+	skipStatusUpdate := false
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
 		// if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
 		// so don't return an error if the device is already removed.
 		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
 		_, ok := err.(ns.NSPathNotExistErr)
+		skipStatusUpdate = true
 		if ok {
-			netnsfound = false
 			logging.Debugf("CmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
 		} else {
-			return cmdErr(nil, "failed to open netns %q: %v", netns, err)
+			logging.Debugf("CmdDel: WARNING failed to open netns %q: %v", netns, err)
 		}
 	}
 
@@ -774,29 +824,12 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return cmdErr(nil, "error getting k8s client: %v", err)
 	}
 
-	pod := (*v1.Pod)(nil)
-	if kubeClient != nil {
-		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-		if err != nil {
-			var waitErr error
-			// in case of a retriable error, retry 10 times with 0.25 sec interval
-			if isCriticalRequestRetriable(err) {
-				waitErr = wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
-					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-					return pod != nil, err
-				})
-				// retry failed, then return error with retry out
-				if waitErr != nil {
-					return cmdErr(k8sArgs, "error getting pod with error: %v", err)
-				}
-			} else if errors.IsNotFound(err) {
-				// If not found, proceed to remove interface with cache
-				pod = nil
-			} else {
-				// Other case, return error
-				return cmdErr(k8sArgs, "error getting pod: %v", err)
-			}
-		}
+	pod, err := getPod(kubeClient, k8sArgs, true)
+	if err != nil {
+		// getPod may be failed but just do print error in its log and continue to delete
+		logging.Errorf("Multus: getPod failed: %v, but continue to delete", err)
+		// skip status update because k8s api seems to be stucked
+		skipStatusUpdate = true
 	}
 
 	// Read the cache to get delegates json for the pod
@@ -831,6 +864,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	} else {
 		defer os.Remove(path)
+		in.Delegates = []*types.DelegateNetConf{}
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
 			return cmdErr(k8sArgs, "failed to load netconf: %v", err)
 		}
@@ -849,12 +883,16 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		if v.ConfListPlugin == true && v.ConfList.CNIVersion == "" && in.CNIVersion != "" {
 			v.ConfList.CNIVersion = in.CNIVersion
 			v.Bytes, err = json.Marshal(v.ConfList)
+			if err != nil {
+				// error happen but continue to delete
+				logging.Errorf("Multus: failed to marshal delegate %q config: %v", v.Name, err)
+			}
 		}
 	}
 
 	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
 	if in.Kubeconfig != "" {
-		if netnsfound {
+		if !skipStatusUpdate {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
 				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
 				if err != nil {
@@ -863,7 +901,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 				}
 			}
 		} else {
-			logging.Debugf("WARNING: Unset SetNetworkStatus skipped due to netns not found.")
+			logging.Debugf("WARNING: Unset SetNetworkStatus skipped")
 		}
 	}
 

pkg/netutils/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package netutils is the package that contains network related utilities.
+package netutils

pkg/netutils/netutils.go

@@ -16,26 +16,24 @@
 package netutils
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"path/filepath"
+
+	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
-	cnitypes "github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"net"
-	"strings"
 )
 
 // DeleteDefaultGW removes the default gateway from marked interfaces.
-func DeleteDefaultGW(args *skel.CmdArgs, ifName string, res *cnitypes.Result) (*current.Result, error) {
-	result, err := current.NewResultFromResult(*res)
-	if err != nil {
-		return nil, logging.Errorf("DeleteDefaultGW: Error creating new from current CNI result: %v", err)
-	}
-
+func DeleteDefaultGW(args *skel.CmdArgs, ifName string) error {
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
-		return nil, logging.Errorf("DeleteDefaultGW: Error getting namespace %v", err)
+		return logging.Errorf("DeleteDefaultGW: Error getting namespace %v", err)
 	}
 	defer netns.Close()
 
@@ -50,40 +48,27 @@ func DeleteDefaultGW(args *skel.CmdArgs, ifName string, res *cnitypes.Result) (*
 		}
 		return err
 	})
-	var newRoutes []*cnitypes.Route
-	for _, route := range result.Routes {
-		if mask, _ := route.Dst.Mask.Size(); mask != 0 {
-			newRoutes = append(newRoutes, route)
-		}
-	}
-	result.Routes = newRoutes
-	return result, err
+	return err
 }
 
 // SetDefaultGW adds a default gateway on a specific interface
-func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cnitypes.Result) (*current.Result, error) {
-
-	// Use the current CNI result...
-	result, err := current.NewResultFromResult(*res)
-	if err != nil {
-		return nil, logging.Errorf("SetDefaultGW: Error creating new CNI result from current: %v", err)
-	}
-
+func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP) error {
 	// This ensures we're acting within the net namespace for the pod.
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
-		return nil, logging.Errorf("SetDefaultGW: Error getting namespace %v", err)
+		return logging.Errorf("SetDefaultGW: Error getting namespace %v", err)
 	}
 	defer netns.Close()
 
-	var newResultDefaultRoutes []*cnitypes.Route
-
 	// Do this within the net namespace.
 	err = netns.Do(func(_ ns.NetNS) error {
 		var err error
 
 		// Pick up the link info as we need the index.
-		link, _ := netlink.LinkByName(ifName)
+		link, err := netlink.LinkByName(ifName)
+		if err != nil {
+			return logging.Errorf("SetDefaultGW: Error getting link %v", err)
+		}
 
 		// Cycle through all the desired gateways.
 		for _, gw := range gateways {
@@ -95,15 +80,6 @@ func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cni
 				Gw:        gw,
 			}
 
-			// Build a new element for the results route
-
-			// Set a correct CIDR depending on IP type
-			_, dstipnet, _ := net.ParseCIDR("::0/0")
-			if strings.Count(gw.String(), ":") < 2 {
-				_, dstipnet, _ = net.ParseCIDR("0.0.0.0/0")
-			}
-			newResultDefaultRoutes = append(newResultDefaultRoutes, &cnitypes.Route{Dst: *dstipnet, GW: gw})
-
 			// Perform the creation of the default route....
 			err = netlink.RouteAdd(&newDefaultRoute)
 			if err != nil {
@@ -113,7 +89,317 @@ func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cni
 		return err
 	})
 
-	result.Routes = newResultDefaultRoutes
-	return result, err
-
+	return err
+}
+
+// DeleteDefaultGWCache updates libcni cache to remove default gateway routes in result
+func DeleteDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, ipv4, ipv6 bool) error {
+	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
+
+	cache, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		return err
+	}
+	logging.Debugf("DeleteDefaultGWCache: update cache to delete GW from: %s", string(cache))
+	newCache, err := deleteDefaultGWCacheBytes(cache, ipv4, ipv6)
+	if err != nil {
+		return err
+	}
+
+	logging.Debugf("DeleteDefaultGWCache: update cache to delete GW: %s", string(newCache))
+	return ioutil.WriteFile(cacheFile, newCache, 0600)
+}
+
+func deleteDefaultGWCacheBytes(cacheFile []byte, ipv4, ipv6 bool) ([]byte, error) {
+	var cachedInfo map[string]interface{}
+	if err := json.Unmarshal(cacheFile, &cachedInfo); err != nil {
+		return nil, err
+	}
+
+	// try to get result
+	_, ok := cachedInfo["result"]
+	if !ok {
+		return nil, fmt.Errorf("cannot get result from cache")
+	}
+
+	resultJSON, ok := cachedInfo["result"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong result type: %v", cachedInfo["result"])
+	}
+	newResult, err := deleteDefaultGWResult(resultJSON, ipv4, ipv6)
+	if err != nil {
+		return nil, err
+	}
+	cachedInfo["result"] = newResult
+
+	newCache, err := json.Marshal(cachedInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode json: %v", err)
+	}
+	return newCache, nil
+}
+
+func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interface{}, error) {
+	for i, r := range routes {
+		route, ok := r.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("wrong route format: %v", r)
+		}
+		_, ok = route["dst"]
+		if ok {
+			dst, ok := route["dst"].(string)
+			if !ok {
+				return nil, fmt.Errorf("wrong dst format: %v", route["dst"])
+			}
+			if dst == dstGW {
+				routes = append(routes[:i], routes[i+1:]...)
+			}
+		}
+	}
+	return routes, nil
+}
+
+func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
+	// try to get cniVersion from result
+	_, ok := result["cniVersion"]
+	if !ok {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return deleteDefaultGWResult020(result, ipv4, ipv6)
+	}
+
+	cniVersion, ok := result["cniVersion"].(string)
+	if !ok {
+		return nil, fmt.Errorf("wrong cniVersion format: %v", result["cniVersion"])
+	}
+
+	if cniVersion == "0.1.0" || cniVersion == "0.2.0" {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return deleteDefaultGWResult020(result, ipv4, ipv6)
+	}
+
+	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+		return nil, fmt.Errorf("not supported version: %s", cniVersion)
+	}
+
+	_, ok = result["routes"]
+	if !ok {
+		// No route in result, hence we do nothing
+		return result, nil
+	}
+	routes, ok := result["routes"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong routes format: %v", result["routes"])
+	}
+
+	var err error
+	// delete IPv4 default routes
+	if ipv4 {
+		routes, err = deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ipv6 {
+		routes, err = deleteDefaultGWResultRoutes(routes, "::0/0")
+		if err != nil {
+			return nil, err
+		}
+	}
+	result["routes"] = routes
+
+	return result, nil
+}
+
+func deleteDefaultGWResult020(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
+	var err error
+	if ipv4 {
+		_, ok := result["ip4"]
+		if ok {
+			ip4, ok := result["ip4"].(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("wrong ip4 format: %v", result["ip4"])
+			}
+
+			_, ok = ip4["routes"]
+			if ok {
+				routes, ok := ip4["routes"].([]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip4 routes format: %v", ip4["routes"])
+				}
+
+				routes, err = deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+				if err != nil {
+					return nil, err
+				}
+				ip4["routes"] = routes
+			}
+		}
+	}
+
+	if ipv6 {
+		_, ok := result["ip6"]
+		if ok {
+			ip6, ok := result["ip6"].(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("wrong ip6 format: %v", result["ip6"])
+			}
+
+			_, ok = ip6["routes"]
+			if ok {
+				routes, ok := ip6["routes"].([]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip6 routes format: %v", ip6["routes"])
+				}
+
+				routes, err = deleteDefaultGWResultRoutes(routes, "::0/0")
+				if err != nil {
+					return nil, err
+				}
+				ip6["routes"] = routes
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// AddDefaultGWCache updates libcni cache to add default gateway result
+func AddDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, gw []net.IP) error {
+	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
+
+	cache, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		return err
+	}
+	logging.Debugf("AddDefaultGWCache: update cache to add GW from: %s", string(cache))
+	newCache, err := addDefaultGWCacheBytes(cache, gw)
+	if err != nil {
+		return err
+	}
+
+	logging.Debugf("AddDefaultGWCache: update cache to add GW: %s", string(newCache))
+	return ioutil.WriteFile(cacheFile, newCache, 0600)
+}
+
+func addDefaultGWCacheBytes(cacheFile []byte, gw []net.IP) ([]byte, error) {
+	var cachedInfo map[string]interface{}
+	if err := json.Unmarshal(cacheFile, &cachedInfo); err != nil {
+		return nil, err
+	}
+
+	// try to get result
+	_, ok := cachedInfo["result"]
+	if !ok {
+		return nil, fmt.Errorf("cannot get result from cache")
+	}
+
+	resultJSON, ok := cachedInfo["result"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong result type: %v", cachedInfo["result"])
+	}
+	newResult, err := addDefaultGWResult(resultJSON, gw)
+	if err != nil {
+		return nil, err
+	}
+	cachedInfo["result"] = newResult
+
+	newCache, err := json.Marshal(cachedInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode json: %v", err)
+	}
+	return newCache, nil
+}
+
+func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
+	// try to get cniVersion from result
+	_, ok := result["cniVersion"]
+	if !ok {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return addDefaultGWResult020(result, gw)
+	}
+
+	cniVersion, ok := result["cniVersion"].(string)
+	if !ok {
+		return nil, fmt.Errorf("wrong cniVersion format: %v", result["cniVersion"])
+	}
+
+	if cniVersion == "0.1.0" || cniVersion == "0.2.0" {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return addDefaultGWResult020(result, gw)
+	}
+
+	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+		return nil, fmt.Errorf("not supported version: %s", cniVersion)
+	}
+
+	routes := []interface{}{}
+	_, ok = result["routes"]
+	if ok {
+		routes, ok = result["routes"].([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("wrong routes format: %v", result["routes"])
+		}
+	}
+
+	for _, g := range gw {
+		dst := "0.0.0.0/0"
+		if g.To4() == nil {
+			dst = "::0/0"
+		}
+		routes = append(routes, map[string]string{
+			"dst": dst,
+			"gw":  g.String(),
+		})
+	}
+	result["routes"] = routes
+
+	return result, nil
+}
+
+func addDefaultGWResult020(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
+	for _, g := range gw {
+		if g.To4() != nil {
+			_, ok := result["ip4"]
+			if ok {
+				ip4, ok := result["ip4"].(map[string]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip4 format: %v", result["ip4"])
+				}
+				routes := []interface{}{}
+				_, ok = ip4["routes"]
+				if ok {
+					routes, ok = ip4["routes"].([]interface{})
+					if !ok {
+						return nil, fmt.Errorf("wrong ip4 routes format: %v", ip4["routes"])
+					}
+				}
+				ip4["routes"] = append(routes, map[string]string{
+					"dst": "0.0.0.0/0",
+					"gw":  g.String(),
+				})
+			}
+		} else {
+			_, ok := result["ip6"]
+			if ok {
+				ip6, ok := result["ip6"].(map[string]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip6 format: %v", result["ip4"])
+				}
+				routes := []interface{}{}
+				_, ok = ip6["routes"]
+				if ok {
+					routes, ok = ip6["routes"].([]interface{})
+					if !ok {
+						return nil, fmt.Errorf("wrong ip6 routes format: %v", ip6["routes"])
+					}
+				}
+				ip6["routes"] = append(routes, map[string]string{
+					"dst": "::/0",
+					"gw":  g.String(),
+				})
+			}
+		}
+	}
+	return result, nil
 }

pkg/testing/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package testing is the package that contains unit-test libraries.
+package testing

pkg/testing/testing.go

@@ -166,3 +166,13 @@ func (r *Result) String() string {
 	}
 	return fmt.Sprintf("%sDNS:%+v", str, r.DNS)
 }
+
+// Int returns a pointer to an int
+func Int(i int) *int {
+	return &i
+}
+
+// Bool returns a pointer to a bool.
+func Bool(b bool) *bool {
+	return &b
+}

pkg/types/conf.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 
 	"github.com/containernetworking/cni/libcni"
@@ -59,9 +60,9 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 }
 
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
-func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
+func LoadDelegateNetConf(bytes []byte, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
-	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
+	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), netElement, deviceID)
 
 	delegateConf := &DelegateNetConf{}
 	if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil {
@@ -79,9 +80,11 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConfList bytes: %v", err)
 			}
+			delegateConf.ResourceName = resourceName
+			delegateConf.DeviceID = deviceID
 		}
-		if net != nil && net.CNIArgs != nil {
-			bytes, err = addCNIArgsInConfList(bytes, net.CNIArgs)
+		if netElement != nil && netElement.CNIArgs != nil {
+			bytes, err = addCNIArgsInConfList(bytes, netElement.CNIArgs)
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add cni-args in NetConfList bytes: %v", err)
 			}
@@ -96,45 +99,51 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			delegateConf.ResourceName = resourceName
 			delegateConf.DeviceID = deviceID
 		}
-		if net != nil && net.CNIArgs != nil {
-			bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs)
+		if netElement != nil && netElement.CNIArgs != nil {
+			bytes, err = addCNIArgsInConfig(bytes, netElement.CNIArgs)
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add cni-args in NetConfList bytes: %v", err)
 			}
 		}
 	}
 
-	if net != nil {
-		if net.Name != "" {
+	if netElement != nil {
+		if netElement.Name != "" {
 			// Overwrite CNI config name with net-attach-def name
-			delegateConf.Name = fmt.Sprintf("%s/%s", net.Namespace, net.Name)
+			delegateConf.Name = fmt.Sprintf("%s/%s", netElement.Namespace, netElement.Name)
 		}
-		if net.InterfaceRequest != "" {
-			delegateConf.IfnameRequest = net.InterfaceRequest
+		if netElement.InterfaceRequest != "" {
+			delegateConf.IfnameRequest = netElement.InterfaceRequest
 		}
-		if net.MacRequest != "" {
-			delegateConf.MacRequest = net.MacRequest
+		if netElement.MacRequest != "" {
+			delegateConf.MacRequest = netElement.MacRequest
 		}
-		if net.IPRequest != nil {
-			delegateConf.IPRequest = net.IPRequest
+		if netElement.IPRequest != nil {
+			delegateConf.IPRequest = netElement.IPRequest
 		}
-		if net.BandwidthRequest != nil {
-			delegateConf.BandwidthRequest = net.BandwidthRequest
+		if netElement.BandwidthRequest != nil {
+			delegateConf.BandwidthRequest = netElement.BandwidthRequest
 		}
-		if net.PortMappingsRequest != nil {
-			delegateConf.PortMappingsRequest = net.PortMappingsRequest
+		if netElement.PortMappingsRequest != nil {
+			delegateConf.PortMappingsRequest = netElement.PortMappingsRequest
 		}
-		if net.GatewayRequest != nil {
-			delegateConf.GatewayRequest = append(delegateConf.GatewayRequest, net.GatewayRequest...)
+		if netElement.GatewayRequest != nil {
+			var list []net.IP
+			if delegateConf.GatewayRequest != nil {
+				list = append(*delegateConf.GatewayRequest, *netElement.GatewayRequest...)
+			} else {
+				list = *netElement.GatewayRequest
+			}
+			delegateConf.GatewayRequest = &list
 		}
-		if net.InfinibandGUIDRequest != "" {
-			delegateConf.InfinibandGUIDRequest = net.InfinibandGUIDRequest
+		if netElement.InfinibandGUIDRequest != "" {
+			delegateConf.InfinibandGUIDRequest = netElement.InfinibandGUIDRequest
 		}
-		if net.DeviceID != "" {
+		if netElement.DeviceID != "" {
 			if deviceID != "" {
 				logging.Debugf("Warning: Both RuntimeConfig and ResourceMap provide deviceID. Ignoring RuntimeConfig")
 			} else {
-				delegateConf.DeviceID = net.DeviceID
+				delegateConf.DeviceID = netElement.DeviceID
 			}
 		}
 	}
@@ -184,7 +193,7 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
 // CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
 // exists, merge data with the runtime config.
 func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
-	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
+	logging.Debugf("CreateCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
 	var cniDeviceInfoFile string
 	var delegateRc *RuntimeConfig
 
@@ -204,21 +213,50 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 	}
 
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
-	// Todo
-	// ingress, egress and bandwidth capability features as same as kubelet.
 	rt := &libcni.RuntimeConf{
 		ContainerID: args.ContainerID,
 		NetNS:       args.Netns,
 		IfName:      ifName,
-		// NOTE: Verbose logging depends on this order, so please keep Args order.
+		// NOTE: Verbose logging (pod namespace/pod name)depends on this order, so please keep Args order.
 		Args: [][2]string{
 			{"IgnoreUnknown", string("true")},
 			{"K8S_POD_NAMESPACE", string(k8sArgs.K8S_POD_NAMESPACE)},
 			{"K8S_POD_NAME", string(k8sArgs.K8S_POD_NAME)},
 			{"K8S_POD_INFRA_CONTAINER_ID", string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)},
+			{"K8S_POD_UID", string(k8sArgs.K8S_POD_UID)},
 		},
 	}
 
+	// Populate rt.Args with CNI_ARGS if the rt.Args value is not set
+	cniArgs := os.Getenv("CNI_ARGS")
+	if cniArgs != "" {
+		for _, arg := range strings.Split(cniArgs, ";") {
+			// SplitN to handle = within values, like BLAH=foo=bar
+			keyval := strings.SplitN(arg, "=", 2)
+			if len(keyval) != 2 {
+				logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %s %s %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval))
+				continue
+			}
+
+			envKey := string(keyval[0])
+			envVal := string(keyval[1])
+			found := false
+			for i := range rt.Args {
+				// Update existing key if its value is empty
+				if rt.Args[i][0] == envKey && rt.Args[i][1] == "" && envVal != "" {
+					logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg)
+					rt.Args[i][1] = envVal
+					found = true
+					break
+				}
+			}
+			if !found {
+				// Add the new key if it didn't exist yet
+				rt.Args = append(rt.Args, [2]string{envKey, envVal})
+			}
+		}
+	}
+
 	if delegateRc != nil {
 		capabilityArgs := map[string]interface{}{}
 		if len(delegateRc.PortMaps) != 0 {
@@ -271,6 +309,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 
 	// Logging
 	logging.SetLogStderr(netconf.LogToStderr)
+	logging.SetLogOptions(netconf.LogOptions)
 	if netconf.LogFile != "" {
 		logging.SetLogFile(netconf.LogFile)
 	}
@@ -505,15 +544,47 @@ func addCNIArgsInConfList(inBytes []byte, cniArgs *map[string]interface{}) ([]by
 	return configBytes, nil
 }
 
-// CheckGatewayConfig check gatewayRequest and mark IsFilterGateway flag if
+// CheckGatewayConfig check gatewayRequest and mark IsFilter{V4,V6}Gateway flag if
 // gw filtering is required
-func CheckGatewayConfig(delegates []*DelegateNetConf) {
-	// Check the Gateway
-	for i, delegate := range delegates {
-		if delegate.GatewayRequest == nil {
-			delegates[i].IsFilterGateway = true
+func CheckGatewayConfig(delegates []*DelegateNetConf) error {
+
+	v4Gateways := 0
+	v6Gateways := 0
+
+	// Check the gateway
+	for _, delegate := range delegates {
+		if delegate.GatewayRequest != nil {
+			for _, gw := range *delegate.GatewayRequest {
+				if gw.To4() != nil {
+					v4Gateways++
+				} else {
+					v6Gateways++
+				}
+			}
 		}
 	}
+
+	if v4Gateways > 1 || v6Gateways > 1 {
+		return fmt.Errorf("multus does not support ECMP for default-route")
+	}
+
+	// set filter flag for each delegate
+	for i, delegate := range delegates {
+		// no GatewayRequest
+		if delegate.GatewayRequest == nil {
+			delegates[i].IsFilterV4Gateway = true
+			delegates[i].IsFilterV6Gateway = true
+		} else {
+			for _, gw := range *delegate.GatewayRequest {
+				if gw.To4() != nil {
+					delegates[i].IsFilterV6Gateway = true
+				} else {
+					delegates[i].IsFilterV4Gateway = true
+				}
+			}
+		}
+	}
+	return nil
 }
 
 // CheckSystemNamespaces checks whether given namespace is in systemNamespaces or not.

pkg/types/conf_test.go

@@ -47,7 +47,6 @@ var _ = Describe("config operations", func() {
 		var err error
 		testNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
-		os.Setenv("CNI_NETNS", testNS.Path())
 		os.Setenv("CNI_PATH", "/some/path")
 
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
@@ -114,27 +113,54 @@ var _ = Describe("config operations", func() {
 
 	It("checks if logFile and logLevel are set correctly", func() {
 		conf := `{
-	    "name": "node-cni-network",
-			"type": "multus",
-			"logLevel": "debug",
-			"logFile": "/var/log/multus.log",
-	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-	    "delegates": [{
-	        "type": "weave-net"
-	    }],
-		"runtimeConfig": {
-	      "portMappings": [
-	        {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
-	      ]
-	    }
-
-	}`
+	"name": "node-cni-network",
+	"type": "multus",
+	"logLevel": "debug",
+	"logFile": "/var/log/multus.log",
+	"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	"delegates": [{
+		"type": "weave-net"
+	}],
+	"runtimeConfig": {
+		"portMappings": [
+			{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+	}
+}`
 		netConf, err := LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 		Expect(netConf.LogLevel).To(Equal("debug"))
 		Expect(netConf.LogFile).To(Equal("/var/log/multus.log"))
 	})
 
+	It("checks if logOptions are set correctly", func() {
+		conf := `{
+	"name": "node-cni-network",
+	"type": "multus",
+	"logOptions": {
+		"maxAge": 5,
+		"maxSize": 100,
+		"maxBackups": 5,
+		"compress": true
+	},
+	"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	"delegates": [{
+		"type": "weave-net"
+	}],
+	"runtimeConfig": {
+		"portMappings": [
+			{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+	}
+}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(*netConf.LogOptions.MaxAge).To(Equal(5))
+		Expect(*netConf.LogOptions.MaxBackups).To(Equal(5))
+		Expect(*netConf.LogOptions.MaxSize).To(Equal(100))
+		Expect(*netConf.LogOptions.Compress).To(Equal(true))
+	})
+
 	It("properly sets namespace isolation using the default namespace", func() {
 		conf := `{
 	    "name": "node-cni-network",
@@ -614,6 +640,44 @@ var _ = Describe("config operations", func() {
 		Expect(rt.CapabilityArgs["portMappings"]).To(Equal(rc.PortMaps))
 	})
 
+	It("creates a valid CNI runtime config with K8s args passed via CNI_ARGS environment variable", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "defaultnetworkwaitseconds": 3,
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    },{
+        "name": "other1",
+        "cniVersion": "0.2.0",
+        "type": "other-plugin"
+    }]
+}`),
+		}
+
+		os.Setenv("CNI_ARGS", "K8S_POD_NAME=dummy;K8S_POD_NAMESPACE=namespacedummy;K8S_POD_INFRA_CONTAINER_ID=123456789;K8S_POD_UID=aaaaa;BLAHBLAH=foo=bar")
+		k8sArgs := &K8sArgs{}
+		rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", &RuntimeConfig{}, nil)
+		fmt.Println("rt.ContainerID: ", rt.ContainerID)
+		Expect(rt.ContainerID).To(Equal("123456789"))
+		Expect(rt.NetNS).To(Equal(args.Netns))
+		Expect(rt.IfName).To(Equal(""))
+		fmt.Println("rt.ContainerID: ", rt.ContainerID)
+		Expect(rt.Args[0]).To(Equal([2]string{"IgnoreUnknown", "true"}))
+		Expect(rt.Args[1]).To(Equal([2]string{"K8S_POD_NAMESPACE", "namespacedummy"}))
+		Expect(rt.Args[2]).To(Equal([2]string{"K8S_POD_NAME", "dummy"}))
+		Expect(rt.Args[3]).To(Equal([2]string{"K8S_POD_INFRA_CONTAINER_ID", "123456789"}))
+		Expect(rt.Args[4]).To(Equal([2]string{"K8S_POD_UID", "aaaaa"}))
+		Expect(rt.Args[5]).To(Equal([2]string{"BLAHBLAH", "foo=bar"}))
+	})
+
 	It("can loadnetworkstatus", func() {
 		result := &types020.Result{
 			CNIVersion: "0.2.0",
@@ -845,4 +909,72 @@ var _ = Describe("config operations", func() {
 		Expect(n.Delegates[0].Name).To(Equal("weave-list"))
 	})
 
+	It("test LoadDelegateNetConf keeps without GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar" }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).To(BeNil())
+	})
+
+	It("test LoadDelegateNetConf keeps empty GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar", "default-route": [] }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).NotTo(BeNil())
+		Expect(len(*netconf.GatewayRequest)).To(BeEquivalentTo(0))
+	})
+
+	It("test LoadDelegateNetConf keeps GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar", "default-route": [ "10.1.1.1" ] }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).NotTo(BeNil())
+		Expect(len(*netconf.GatewayRequest)).To(BeEquivalentTo(1))
+	})
+
 })

pkg/types/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package types contains common types in the multus.
+package types

pkg/types/types.go

@@ -16,6 +16,7 @@
 package types
 
 import (
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
@@ -44,6 +45,7 @@ type NetConf struct {
 	LogFile         string                   `json:"logFile"`
 	LogLevel        string                   `json:"logLevel"`
 	LogToStderr     bool                     `json:"logToStderr,omitempty"`
+	LogOptions      *logging.LogOptions      `json:"logOptions,omitempty"`
 	RuntimeConfig   *RuntimeConfig           `json:"runtimeConfig,omitempty"`
 	// Default network readiness options
 	ReadinessIndicatorFile string `json:"readinessindicatorfile"`
@@ -97,8 +99,9 @@ type DelegateNetConf struct {
 	IPRequest             []string        `json:"ipRequest,omitempty"`
 	PortMappingsRequest   []*PortMapEntry `json:"-"`
 	BandwidthRequest      *BandwidthEntry `json:"-"`
-	GatewayRequest        []net.IP        `json:"default-route,omitempty"`
-	IsFilterGateway       bool
+	GatewayRequest        *[]net.IP       `json:"default-route,omitempty"`
+	IsFilterV4Gateway     bool
+	IsFilterV6Gateway     bool
 	// MasterPlugin is only used internal housekeeping
 	MasterPlugin bool `json:"-"`
 	// Conflist plugin is only used internal housekeeping
@@ -147,7 +150,7 @@ type NetworkSelectionElement struct {
 	// CNIArgs contains additional CNI arguments for the network interface
 	CNIArgs *map[string]interface{} `json:"cni-args"`
 	// GatewayRequest contains default route IP address for the pod
-	GatewayRequest []net.IP `json:"default-route,omitempty"`
+	GatewayRequest *[]net.IP `json:"default-route,omitempty"`
 }
 
 // K8sArgs is the valid CNI_ARGS used for Kubernetes
@@ -157,6 +160,7 @@ type K8sArgs struct {
 	K8S_POD_NAME               types.UnmarshallableString //revive:disable-line
 	K8S_POD_NAMESPACE          types.UnmarshallableString //revive:disable-line
 	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString //revive:disable-line
+	K8S_POD_UID                types.UnmarshallableString //revive:disable-line
 }
 
 // ResourceInfo is struct to hold Pod device allocation information

vendor/github.com/Microsoft/go-winio/go.mod

@@ -1,9 +0,0 @@
-module github.com/Microsoft/go-winio
-
-go 1.12
-
-require (
-	github.com/pkg/errors v0.8.1
-	github.com/sirupsen/logrus v1.4.1
-	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
-)

vendor/github.com/Microsoft/go-winio/go.sum

@@ -1,18 +0,0 @@
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/cespare/xxhash/v2/go.mod

@@ -1,3 +0,0 @@
-module github.com/cespare/xxhash/v2
-
-go 1.11

vendor/github.com/fsnotify/fsnotify/go.mod

@@ -1,5 +0,0 @@
-module github.com/fsnotify/fsnotify
-
-go 1.13
-
-require golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9

vendor/github.com/fsnotify/fsnotify/go.sum

@@ -1,2 +0,0 @@
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/go-logr/logr/go.mod

@@ -1,3 +0,0 @@
-module github.com/go-logr/logr
-
-go 1.14

vendor/github.com/google/gofuzz/go.mod

@@ -1,3 +0,0 @@
-module github.com/google/gofuzz
-
-go 1.12

vendor/github.com/json-iterator/go/go.mod

@@ -1,11 +0,0 @@
-module github.com/json-iterator/go
-
-go 1.12
-
-require (
-	github.com/davecgh/go-spew v1.1.1
-	github.com/google/gofuzz v1.0.0
-	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421
-	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
-	github.com/stretchr/testify v1.3.0
-)

vendor/github.com/json-iterator/go/go.sum

@@ -1,14 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

vendor/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1/types.go

@@ -43,7 +43,7 @@ const (
 	DeviceInfoTypeVHostUser = "vhost-user"
 	DeviceInfoTypeMemif     = "memif"
 	DeviceInfoTypeVDPA      = "vdpa"
-	DeviceInfoVersion       = "1.0.0"
+	DeviceInfoVersion       = "1.1.0"
 )
 
 // DeviceInfo contains the information of the device associated
@@ -58,18 +58,20 @@ type DeviceInfo struct {
 }
 
 type PciDevice struct {
-	PciAddress   string `json:"pci-address,omitempty"`
-	Vhostnet     string `json:"vhost-net,omitempty"`
-	RdmaDevice   string `json:"rdma-device,omitempty"`
-	PfPciAddress string `json:"pf-pci-address,omitempty"`
+	PciAddress        string `json:"pci-address,omitempty"`
+	Vhostnet          string `json:"vhost-net,omitempty"`
+	RdmaDevice        string `json:"rdma-device,omitempty"`
+	PfPciAddress      string `json:"pf-pci-address,omitempty"`
+	RepresentorDevice string `json:"representor-device,omitempty"`
 }
 
 type VdpaDevice struct {
-	ParentDevice string `json:"parent-device,omitempty"`
-	Driver       string `json:"driver,omitempty"`
-	Path         string `json:"path,omitempty"`
-	PciAddress   string `json:"pci-address,omitempty"`
-	PfPciAddress string `json:"pf-pci-address,omitempty"`
+	ParentDevice      string `json:"parent-device,omitempty"`
+	Driver            string `json:"driver,omitempty"`
+	Path              string `json:"path,omitempty"`
+	PciAddress        string `json:"pci-address,omitempty"`
+	PfPciAddress      string `json:"pf-pci-address,omitempty"`
+	RepresentorDevice string `json:"representor-device,omitempty"`
 }
 
 const (

vendor/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils/net-attach-def.go

@@ -143,13 +143,7 @@ func CreateNetworkStatus(r cnitypes.Result, networkName string, defaultNetwork b
 	}
 
 	for _, ipconfig := range result.IPs {
-		if ipconfig.Version == "4" && ipconfig.Address.IP.To4() != nil {
-			netStatus.IPs = append(netStatus.IPs, ipconfig.Address.IP.String())
-		}
-
-		if ipconfig.Version == "6" && ipconfig.Address.IP.To16() != nil {
-			netStatus.IPs = append(netStatus.IPs, ipconfig.Address.IP.String())
-		}
+		netStatus.IPs = append(netStatus.IPs, ipconfig.Address.IP.String())
 	}
 
 	v1dns := convertDNS(result.DNS)

vendor/github.com/nxadm/tail/go.mod

@@ -1,9 +0,0 @@
-module github.com/nxadm/tail
-
-go 1.13
-
-require (
-	github.com/fsnotify/fsnotify v1.4.7
-	golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
-)

vendor/github.com/nxadm/tail/go.sum

@@ -1,6 +0,0 @@
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

vendor/github.com/onsi/ginkgo/go.mod

@@ -1,9 +0,0 @@
-module github.com/onsi/ginkgo
-
-require (
-	github.com/nxadm/tail v1.4.4
-	github.com/onsi/gomega v1.7.1
-	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e
-)
-
-go 1.12

vendor/github.com/onsi/ginkgo/go.sum

@@ -1,29 +0,0 @@
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/onsi/gomega/go.mod

@@ -1,11 +0,0 @@
-module github.com/onsi/gomega
-
-go 1.14
-
-require (
-	github.com/golang/protobuf v1.4.2
-	github.com/onsi/ginkgo v1.12.1
-	golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
-	gopkg.in/yaml.v2 v2.3.0
-)

vendor/github.com/onsi/gomega/go.sum

@@ -1,69 +0,0 @@
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/prometheus/procfs/go.mod

@@ -1,9 +0,0 @@
-module github.com/prometheus/procfs
-
-go 1.12
-
-require (
-	github.com/google/go-cmp v0.3.1
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e
-)

vendor/github.com/prometheus/procfs/go.sum

@@ -1,6 +0,0 @@
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e h1:LwyF2AFISC9nVbS6MgzsaQNSUsRXI49GS+YQ5KX/QH0=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/spf13/pflag/go.mod

@@ -1,3 +0,0 @@
-module github.com/spf13/pflag
-
-go 1.12

vendor/github.com/vishvananda/netlink/go.mod

@@ -1,8 +0,0 @@
-module github.com/vishvananda/netlink
-
-go 1.12
-
-require (
-	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
-	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
-)

vendor/github.com/vishvananda/netlink/go.sum

@@ -1,6 +0,0 @@
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867 h1:JoRuNIf+rpHl+VhScRQQvzbHed86tKkqwPMV34T8myw=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 h1:sIky/MyNRSHTrdxfsiUSS4WIAMvInbeXljJz+jDjeYE=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/vishvananda/netns/go.mod

@@ -1,5 +0,0 @@
-module github.com/vishvananda/netns
-
-go 1.12
-
-require golang.org/x/sys v0.0.0-20200217220822-9197077df867

vendor/github.com/vishvananda/netns/go.sum

@@ -1,2 +0,0 @@
-golang.org/x/sys v0.0.0-20200217220822-9197077df867 h1:JoRuNIf+rpHl+VhScRQQvzbHed86tKkqwPMV34T8myw=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/golang.org/x/crypto/ssh/terminal/util_linux.go

@@ -1,10 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package terminal
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TCGETS
-const ioctlWriteTermios = unix.TCSETS

vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go

@@ -1,58 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package terminal provides support functions for dealing with terminals, as
-// commonly found on UNIX systems.
-//
-// Putting a terminal into raw mode is the most common requirement:
-//
-// 	oldState, err := terminal.MakeRaw(0)
-// 	if err != nil {
-// 	        panic(err)
-// 	}
-// 	defer terminal.Restore(0, oldState)
-package terminal
-
-import (
-	"fmt"
-	"runtime"
-)
-
-type State struct{}
-
-// IsTerminal returns whether the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
-	return false
-}
-
-// MakeRaw put the terminal connected to the given file descriptor into raw
-// mode and returns the previous state of the terminal so that it can be
-// restored.
-func MakeRaw(fd int) (*State, error) {
-	return nil, fmt.Errorf("terminal: MakeRaw not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
-}
-
-// GetState returns the current state of a terminal which may be useful to
-// restore the terminal after a signal.
-func GetState(fd int) (*State, error) {
-	return nil, fmt.Errorf("terminal: GetState not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
-}
-
-// Restore restores the terminal connected to the given file descriptor to a
-// previous state.
-func Restore(fd int, state *State) error {
-	return fmt.Errorf("terminal: Restore not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
-}
-
-// GetSize returns the dimensions of the given terminal.
-func GetSize(fd int) (width, height int, err error) {
-	return 0, 0, fmt.Errorf("terminal: GetSize not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
-}
-
-// ReadPassword reads a line of input from a terminal without local echo.  This
-// is commonly used for inputting passwords and other sensitive data. The slice
-// returned does not include the \n.
-func ReadPassword(fd int) ([]byte, error) {
-	return nil, fmt.Errorf("terminal: ReadPassword not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
-}

vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go

@@ -1,124 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build solaris
-
-package terminal // import "golang.org/x/crypto/ssh/terminal"
-
-import (
-	"golang.org/x/sys/unix"
-	"io"
-	"syscall"
-)
-
-// State contains the state of a terminal.
-type State struct {
-	termios unix.Termios
-}
-
-// IsTerminal returns whether the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
-	_, err := unix.IoctlGetTermio(fd, unix.TCGETA)
-	return err == nil
-}
-
-// ReadPassword reads a line of input from a terminal without local echo.  This
-// is commonly used for inputting passwords and other sensitive data. The slice
-// returned does not include the \n.
-func ReadPassword(fd int) ([]byte, error) {
-	// see also: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libast/common/uwin/getpass.c
-	val, err := unix.IoctlGetTermios(fd, unix.TCGETS)
-	if err != nil {
-		return nil, err
-	}
-	oldState := *val
-
-	newState := oldState
-	newState.Lflag &^= syscall.ECHO
-	newState.Lflag |= syscall.ICANON | syscall.ISIG
-	newState.Iflag |= syscall.ICRNL
-	err = unix.IoctlSetTermios(fd, unix.TCSETS, &newState)
-	if err != nil {
-		return nil, err
-	}
-
-	defer unix.IoctlSetTermios(fd, unix.TCSETS, &oldState)
-
-	var buf [16]byte
-	var ret []byte
-	for {
-		n, err := syscall.Read(fd, buf[:])
-		if err != nil {
-			return nil, err
-		}
-		if n == 0 {
-			if len(ret) == 0 {
-				return nil, io.EOF
-			}
-			break
-		}
-		if buf[n-1] == '\n' {
-			n--
-		}
-		ret = append(ret, buf[:n]...)
-		if n < len(buf) {
-			break
-		}
-	}
-
-	return ret, nil
-}
-
-// MakeRaw puts the terminal connected to the given file descriptor into raw
-// mode and returns the previous state of the terminal so that it can be
-// restored.
-// see http://cr.illumos.org/~webrev/andy_js/1060/
-func MakeRaw(fd int) (*State, error) {
-	termios, err := unix.IoctlGetTermios(fd, unix.TCGETS)
-	if err != nil {
-		return nil, err
-	}
-
-	oldState := State{termios: *termios}
-
-	termios.Iflag &^= unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON
-	termios.Oflag &^= unix.OPOST
-	termios.Lflag &^= unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN
-	termios.Cflag &^= unix.CSIZE | unix.PARENB
-	termios.Cflag |= unix.CS8
-	termios.Cc[unix.VMIN] = 1
-	termios.Cc[unix.VTIME] = 0
-
-	if err := unix.IoctlSetTermios(fd, unix.TCSETS, termios); err != nil {
-		return nil, err
-	}
-
-	return &oldState, nil
-}
-
-// Restore restores the terminal connected to the given file descriptor to a
-// previous state.
-func Restore(fd int, oldState *State) error {
-	return unix.IoctlSetTermios(fd, unix.TCSETS, &oldState.termios)
-}
-
-// GetState returns the current state of a terminal which may be useful to
-// restore the terminal after a signal.
-func GetState(fd int) (*State, error) {
-	termios, err := unix.IoctlGetTermios(fd, unix.TCGETS)
-	if err != nil {
-		return nil, err
-	}
-
-	return &State{termios: *termios}, nil
-}
-
-// GetSize returns the dimensions of the given terminal.
-func GetSize(fd int) (width, height int, err error) {
-	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
-	if err != nil {
-		return 0, 0, err
-	}
-	return int(ws.Col), int(ws.Row), nil
-}