Provide the option to not to log via STDERR

Today, Multus will always log via STDERR, and these logs will then
logged by the Kubelet. If we also choose to have Multus log to a file by
setting the LogFile option in the CNI configuration, the same logs will
be logged twice.

This commit provide the option to disable logging to STDERR.

Signed-off-by: Yun Zhou <yunz@nvidia.com>

.github/workflows/build.yml

@@ -0,0 +1,24 @@
+on: [push, pull_request]
+name: Build
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+        goarch: [386, amd64, arm, arm64, ppc64le, s390x]
+        os: [ubuntu-latest] #, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Build
+      env:
+        GOARCH: ${{ matrix.goarch }}
+        GOOS: ${{ matrix.goos }}
+      run: ./hack/build-go.sh

.github/workflows/docker-build-push-master.yml

@@ -1,64 +0,0 @@
-name: Docker-build-push-latest
-on: 
-  push:
-    branches:
-      - master
-jobs:
-  build:
-    name: Docker build
-    runs-on: ubuntu-latest
-    env:
-      DOCKER_CLI_EXPERIMENTAL: enabled
-      GO111MODULE: on
-      TARGET: amd64
-      REPOSITORY: nfvpe/multus
-      REPOSITORY_USER: nfvperobot
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: Build latest-amd64
-        run: docker build -t ${REPOSITORY}:latest-amd64 .
-  
-      - name: Build latest-ppc64le
-        run: docker build -t ${REPOSITORY}:latest-ppc64le -f Dockerfile.ppc64le .
-  
-      - name: Build latest-arm64v8
-        run: docker build -t ${REPOSITORY}:latest-arm64v8 -f Dockerfile.arm64 .
-  
-      - name: Build latest-origin
-        run: docker build -t ${REPOSITORY}:latest-origin -f Dockerfile.openshift .
-  
-      - name: Tag snapshot
-        run: |
-          docker tag ${REPOSITORY}:latest-amd64 ${REPOSITORY}:snapshot-amd64
-          docker tag ${REPOSITORY}:latest-ppc64le ${REPOSITORY}:snapshot-ppc64le
-          docker tag ${REPOSITORY}:latest-arm64v8 ${REPOSITORY}:snapshot-arm64v8
-
-      - name: Login to registry
-        if: github.repository == 'intel/multus-cni'
-        run: docker login -u ${REPOSITORY_USER} -p ${{ secrets.REPOSITORY_PASS }}
-  
-      - name: Push latest/snapshot images
-        if: github.repository == 'intel/multus-cni'
-        run: |
-          docker push ${REPOSITORY}:latest-amd64
-          docker push ${REPOSITORY}:latest-ppc64le
-          docker push ${REPOSITORY}:latest-arm64v8
-          docker push ${REPOSITORY}:snapshot-amd64
-          docker push ${REPOSITORY}:snapshot-ppc64le
-          docker push ${REPOSITORY}:snapshot-arm64v8
-  
-      - name: Create manifest for multi-arch images
-        if: github.repository == 'intel/multus-cni'
-        run: |
-          docker manifest create ${REPOSITORY}:snapshot ${REPOSITORY}:snapshot-amd64 ${REPOSITORY}:snapshot-ppc64le ${REPOSITORY}:snapshot-arm64v8
-          docker manifest annotate ${REPOSITORY}:snapshot ${REPOSITORY}:snapshot-amd64 --arch amd64
-          docker manifest annotate ${REPOSITORY}:snapshot ${REPOSITORY}:snapshot-ppc64le --arch ppc64le
-          docker manifest annotate ${REPOSITORY}:snapshot ${REPOSITORY}:snapshot-arm64v8 --arch arm64
-          docker manifest push ${REPOSITORY}:snapshot
-          docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:latest-amd64 ${REPOSITORY}:latest-ppc64le ${REPOSITORY}:latest-arm64v8
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-amd64 --arch amd64
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-ppc64le --arch ppc64le
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-arm64v8 --arch arm64
-          docker manifest push ${REPOSITORY}:latest

.github/workflows/docker-build-push-release.yml

@@ -1,75 +0,0 @@
-name: Docker-build-push-stable
-on: 
-  push:
-    tags:
-      - v*
-jobs:
-  build:
-    name: Docker build
-    runs-on: ubuntu-latest
-    env:
-      DOCKER_CLI_EXPERIMENTAL: enabled
-      GO111MODULE: on
-      TARGET: amd64
-      REPOSITORY: nfvpe/multus
-      REPOSITORY_USER: nfvperobot
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: Build latest-amd64
-        run: docker build -t ${REPOSITORY}:latest-amd64 .
-  
-      - name: Build latest-ppc64le
-        run: docker build -t ${REPOSITORY}:latest-ppc64le -f Dockerfile.ppc64le .
-  
-      - name: Build latest-arm64v8
-        run: docker build -t ${REPOSITORY}:latest-arm64v8 -f Dockerfile.arm64 .
-  
-      - name: Build latest-origin
-        run: docker build -t ${REPOSITORY}:latest-origin -f Dockerfile.openshift .
-  
-      - name: Tag stable
-        run: |
-          docker tag ${REPOSITORY}:latest-amd64 ${REPOSITORY}:stable-amd64
-          docker tag ${REPOSITORY}:latest-ppc64le ${REPOSITORY}:stable-ppc64le
-          docker tag ${REPOSITORY}:latest-arm64v8 ${REPOSITORY}:stable-arm64v8
-          docker tag ${REPOSITORY}:latest-amd64 ${REPOSITORY}:${GITHUB_REF##*/}-amd64
-          docker tag ${REPOSITORY}:latest-ppc64le ${REPOSITORY}:${GITHUB_REF##*/}-ppc64le
-          docker tag ${REPOSITORY}:latest-arm64v8 ${REPOSITORY}:${GITHUB_REF##*/}-arm64v8
-  
-      - name: Login to registry
-        if: github.repository == 'intel/multus-cni'
-        run: docker login -u ${REPOSITORY_USER} -p ${{ secrets. REPOSITORY_PASS }}
-  
-      - name: Push latest/snapshot images
-        if: github.repository == 'intel/multus-cni'
-        run: |
-          docker push ${REPOSITORY}:latest-amd64
-          docker push ${REPOSITORY}:latest-ppc64le
-          docker push ${REPOSITORY}:latest-arm64v8
-          docker push ${REPOSITORY}:stable-amd64
-          docker push ${REPOSITORY}:stable-ppc64le
-          docker push ${REPOSITORY}:stable-arm64v8
-          docker push ${REPOSITORY}:${GITHUB_REF##*/}-amd64
-          docker push ${REPOSITORY}:${GITHUB_REF##*/}-ppc64le
-          docker push ${REPOSITORY}:${GITHUB_REF##*/}-arm64v8
-  
-      - name: Create manifest for multi-arch images
-        if: github.repository == 'intel/multus-cni'
-        run: |
-          docker manifest create ${REPOSITORY}:stable ${REPOSITORY}:stable-amd64 ${REPOSITORY}:stable-ppc64le ${REPOSITORY}:stable-arm64v8
-          docker manifest annotate ${REPOSITORY}:stable ${REPOSITORY}:stable-amd64 --arch amd64
-          docker manifest annotate ${REPOSITORY}:stable ${REPOSITORY}:stable-ppc64le --arch ppc64le
-          docker manifest annotate ${REPOSITORY}:stable ${REPOSITORY}:stable-arm64v8 --arch arm64
-          docker manifest push ${REPOSITORY}:stable
-          docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:latest-amd64 ${REPOSITORY}:latest-ppc64le ${REPOSITORY}:latest-arm64v8
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-amd64 --arch amd64
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-ppc64le --arch ppc64le
-          docker manifest annotate ${REPOSITORY}:latest ${REPOSITORY}:latest-arm64v8 --arch arm64
-          docker manifest push ${REPOSITORY}:latest
-          docker manifest create ${REPOSITORY}:${GITHUB_REF##*/} ${REPOSITORY}:${GITHUB_REF##*/}-amd64 ${REPOSITORY}:${GITHUB_REF##*/}-ppc64le ${REPOSITORY}:${GITHUB_REF##*/}-arm64v8
-          docker manifest annotate ${REPOSITORY}:${GITHUB_REF##*/} ${REPOSITORY}:${GITHUB_REF##*/}-amd64 --arch amd64
-          docker manifest annotate ${REPOSITORY}:${GITHUB_REF##*/} ${REPOSITORY}:${GITHUB_REF##*/}-ppc64le --arch ppc64le
-          docker manifest annotate ${REPOSITORY}:${GITHUB_REF##*/} ${REPOSITORY}:${GITHUB_REF##*/}-arm64v8 --arch arm64
-          docker manifest push ${REPOSITORY}:${GITHUB_REF##*/}

.github/workflows/docker-build.yml

@@ -1,22 +0,0 @@
-name: Docker-build
-on: [pull_request]
-jobs:
-  build:
-    name: Docker build
-    runs-on: ubuntu-latest
-    env:
-      DOCKER_CLI_EXPERIMENTAL: enabled
-      GO111MODULE: on
-      REPOSITORY: nfvpe/multus
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: Build latest-amd64
-        run: docker build -t ${REPOSITORY}:latest-amd64 .
-  
-      - name: Build latest-ppc64le
-        run: docker build -t ${REPOSITORY}:latest-ppc64le -f Dockerfile.ppc64le .
-  
-      - name: Build latest-origin
-        run: docker build -t ${REPOSITORY}:latest-origin -f Dockerfile.openshift .

.github/workflows/go-build-arm64.yml

@@ -1,25 +0,0 @@
-name: Go-build-ppc64le
-on: [push, pull_request]
-jobs:
-
-  build:
-    name: Build and test
-    runs-on: ubuntu-latest
-    if: >
-      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
-        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
-    env:
-      GO111MODULE: on
-      TARGET: arm64
-    steps:
-      - name: Set up Go 1.13
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.13
-        id: go
-  
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: Build
-        run: GOARCH="${TARGET}" ./build

.github/workflows/go-build-ppc64.yml

@@ -1,25 +0,0 @@
-name: Go-build-ppc64le
-on: [push, pull_request]
-jobs:
-
-  build:
-    name: Build and test
-    runs-on: ubuntu-latest
-    if: >
-      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
-        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
-    env:
-      GO111MODULE: on
-      TARGET: ppc64le
-    steps:
-      - name: Set up Go 1.13
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.13
-        id: go
-  
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: Build
-        run: GOARCH="${TARGET}" ./build

.github/workflows/go-build-test-amd64.yml

@@ -1,48 +0,0 @@
-name: Go-build-and-test-amd64
-on: [push, pull_request]
-jobs:
-
-  build:
-    name: Build and test
-    runs-on: ubuntu-latest
-    if: >
-      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
-        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
-    env:
-      GO111MODULE: on
-      TARGET: amd64
-    steps:
-      - name: Set up Go 1.13
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.13
-        id: go
-  
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-  
-      - name: install goberalls
-        run: go get github.com/mattn/goveralls
-  
-      - name: install golint
-        run: go get -u golang.org/x/lint/golint
-  
-      - name: golint
-        run: golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
-  
-      - name: gofmt
-        run: go fmt ./...
-  
-      - name: go vet
-        run: go vet ./...
-  
-      - name: Build
-        run: GOARCH="${TARGET}" ./build
-  
-      - name: Go test
-        run: sudo ./test.sh
-  
-      - name: goveralls
-        uses: shogo82148/actions-goveralls@v1
-        with:
-          path-to-profile: coverage.out

.github/workflows/image-build.yml

@@ -0,0 +1,92 @@
+name: Image build
+on: [pull_request]
+jobs:
+  build-amd64:
+    name: Image build/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-amd64
+          file: deployments/Dockerfile
+
+  build-arm64:
+    name: Image build/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm64
+          file: deployments/Dockerfile.arm64
+
+  build-ppc64le:
+    name: Image build/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-ppc64le
+          file: deployments/Dockerfile.ppc64le
+
+  build-s390:
+    name: Image build/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-s390x
+          file: deployments/Dockerfile.s390x
+
+  build-origin:
+    name: Image build/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-origin
+          file: deployments/Dockerfile.openshift

.github/workflows/image-push-master.yml

@@ -0,0 +1,187 @@
+name: Image push for master
+on: 
+  push:
+    branches:
+      - master
+env:
+  REPOSITORY: docker.io/nfvpe/multus
+  REPOSITORY_USER: nfvperobot
+jobs:
+  push-amd64:
+    name: Image push/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:latest-amd64
+            ${{ env.REPOSITORY }}:snapshot-amd64
+          file: deployments/Dockerfile
+
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:latest-arm64
+            ${{ env.REPOSITORY }}:snapshot-arm64
+          file: deployments/Dockerfile.arm64
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:latest-ppc64le
+            ${{ env.REPOSITORY }}:snapshot-ppc64le
+          file: deployments/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:latest-s390x
+            ${{ env.REPOSITORY }}:snapshot-s390x
+          file: deployments/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:latest-origin
+            ${{ env.REPOSITORY }}:snapshot-origin
+          file: deployments/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'intel'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ env.REPOSITORY }}:snapshot-amd64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm64
+          docker pull ${{ env.REPOSITORY }}:snapshot-ppc64le
+          docker pull ${{ env.REPOSITORY }}:snapshot-s390x
+          docker pull ${{ env.REPOSITORY }}:latest-amd64
+          docker pull ${{ env.REPOSITORY }}:latest-arm64
+          docker pull ${{ env.REPOSITORY }}:latest-ppc64le
+          docker pull ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:snapshot
+          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:latest

.github/workflows/image-push-release.yml

@@ -0,0 +1,229 @@
+name: Image push release
+on: 
+  push:
+    tags:
+      - v*
+env:
+  REPOSITORY: docker.io/nfvpe/multus
+  REPOSITORY_USER: nfvperobot
+jobs:
+  push-amd64:
+    name: Image push/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:stable-amd64
+            ${{ steps.docker_meta.outputs.tags }}-amd64
+          file: deployments/Dockerfile
+
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:stable-arm64
+            ${{ steps.docker_meta.outputs.tags }}-arm64
+          file: deployments/Dockerfile.arm64
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:stable-ppc64le
+            ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          file: deployments/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:stable-s390x
+            ${{ steps.docker_meta.outputs.tags }}-s390x
+          file: deployments/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'intel'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REPOSITORY }}:stable-origin
+            ${{ steps.docker_meta.outputs.tags }}-origin
+          file: deployments/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.REPOSITORY }}
+          tag-latest: false
+
+      - name: Login to Container Registry
+        if: github.repository_owner == 'intel'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.REPOSITORY_USER }}
+          password: ${{ secrets.REPOSITORY_PASS }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'intel'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ steps.docker_meta.outputs.tags }}-amd64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          docker pull ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 --arch amd64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm64 --arch arm64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-ppc64le --arch ppc64le
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-s390x --arch s390x
+          docker manifest push ${{ steps.docker_meta.outputs.tags }}
+          docker pull ${{ env.REPOSITORY }}:stable-amd64
+          docker pull ${{ env.REPOSITORY }}:stable-arm64
+          docker pull ${{ env.REPOSITORY }}:stable-ppc64le
+          docker pull ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:stable

.github/workflows/kind-e2e.yml

@@ -1,9 +1,7 @@
 name: e2e-kind
 on: [push, pull_request]
 jobs:
-
-  build:
-    name: Build and test
+  e2e-kind:
     runs-on: ubuntu-latest
     if: >
       (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
@@ -16,7 +14,7 @@ jobs:
         run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
   
       - name: Build latest-amd64
-        run: docker build -t localhost:5000/multus:e2e .
+        run: docker build -t localhost:5000/multus:e2e -f deployments/Dockerfile .
  
       - name: Push to local registry
         run: docker push localhost:5000/multus:e2e
@@ -33,6 +31,10 @@ jobs:
         working-directory: ./e2e
         run: ./test-simple-macvlan1.sh
 
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
       - name: cleanup cluster and registry
         run: |
           kind delete cluster

.github/workflows/release.yml

@@ -1,27 +1,24 @@
-name: Release-binaries
-
+name: Release binaries
 on:
   push:
     tags:
-      - v*
-
+      - 'v*'
 jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
-      - name: Unshallow
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v2
         with:
-          go-version: 1.13.x
+          go-version: 1.15.x
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v1
+        uses: goreleaser/goreleaser-action@v2
         with:
           version: latest
           args: release --rm-dist

.github/workflows/test.yml

@@ -0,0 +1,49 @@
+on: [push, pull_request]
+name: Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Run Revive
+      run: |
+        GO111MODULE=off go get github.com/mgechev/revive
+        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./... # this is ouput for user
+        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./...| xargs -0 -r false # this is for github actions
+
+    - name: Run go fmt
+      run: go fmt ./...
+      #run: diff -u <(echo -n) <(gofmt -d -s .)
+
+    - name: Run go vet
+      run: go vet ./...
+
+    - name: Test
+      run: sudo ./hack/test-go.sh
+
+    - name: Send coverage
+      uses: shogo82148/actions-goveralls@v1
+      with:
+        path-to-profile: coverage.out
+        flag-name: Go-${{ matrix.go }}
+        parallel: true
+
+  # notifies that all test jobs are finished.
+  finish:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true

.gitignore

@@ -1,5 +1,6 @@
 # Binary output dir
 bin/
+e2e/bin/
 
 # GOPATH created by the build script
 gopath/

.goreleaser.yml

@@ -9,7 +9,7 @@ builds:
 -  
   env:
   - CGO_ENABLED=0
-  main: ./multus/
+  main: ./cmd/
   goos:
   - linux
   goarch:
@@ -17,6 +17,7 @@ builds:
   - amd64
   - arm
   - arm64
+  - s390x
 archives:
   - wrap_in_directory: true
 checksum:

.travis.yml

@@ -41,10 +41,10 @@ before_script:
 #  - gocyclo -over 15 ./multus
 
 script:
-  - GOARCH="${TARGET}" ./build
+  - GOARCH="${TARGET}" ./hack/build-go.sh
   - |
     if [ "${TARGET}" == "amd64" ]; then
-      sudo env PATH=${PATH} ./test.sh
+      sudo env PATH=${PATH} ./scripts/test.sh
       goveralls -coverprofile=coverage.out -service=travis-ci
       docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 .
       docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le -f Dockerfile.ppc64le .

CODE_OF_CONDUCT.md

@@ -0,0 +1,130 @@
+# Multus CNI Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[The Multus Slack Page](https://intel-corp.herokuapp.com/).
+All complaints will be reviewed and investigated promptly and fairly. Or you
+may specifically contact Doug Smith (dosmith@redhat.com) via email.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
+

README.md

@@ -1,8 +1,8 @@
 # Multus-CNI
 
-![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png)
+![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/docs/images/Multus.png)
 
-[![Travis CI](https://travis-ci.org/intel/multus-cni.svg?branch=master)](https://travis-ci.org/intel/multus-cni/builds)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/intel/multus-cni/badge.svg)](https://coveralls.io/github/intel/multus-cni)
+[![Build](https://github.com/intel/multus-cni/actions/workflows/build.yml/badge.svg)](https://github.com/intel/multus-cni/actions/workflows/build.yml)[![Test](https://github.com/intel/multus-cni/actions/workflows/test.yml/badge.svg)](https://github.com/intel/multus-cni/actions/workflows/test.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/intel/multus-cni/badge.svg)](https://coveralls.io/github/intel/multus-cni)
 
 Multus CNI enables attaching multiple network interfaces to pods in Kubernetes.
 
@@ -18,11 +18,11 @@ Multus is one of the projects in the [Baremetal Container Experience kit](https:
 
 Here's an illustration of the network interfaces attached to a pod, as provisioned by Multus CNI. The diagram shows the pod with three interfaces: `eth0`, `net0` and `net1`. `eth0` connects kubernetes cluster network to connect with kubernetes server/services (e.g. kubernetes api-server, kubelet and so on). `net0` and `net1` are additional network attachments and connect to other networks by using [other CNI plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) (e.g. vlan/vxlan/ptp).
 
-![multus-pod-image](doc/images/multus-pod-image.svg)
+![multus-pod-image](docs/images/multus-pod-image.svg)
 
 ## Quickstart Installation Guide
 
-The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be creatd with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](doc/quickstart.md).
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be creatd with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
 
 Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
 
@@ -30,22 +30,22 @@ Clone this GitHub repository, we'll apply a daemonset which installs Multus usin
 $ cat ./images/multus-daemonset.yml | kubectl apply -f -
 ```
 
-This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](doc/quickstart.md)
+This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
 
 ## Additional installation Options
 
 - Install via daemonset using the quick-start guide, above.
 - Download binaries from [release page](https://github.com/intel/multus-cni/releases)
 - By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
-- Or, roll-you-own and build from source
-  - See [Development](doc/development.md)
+- Or, roll-your-own and build from source
+  - See [Development](docs/development.md)
 
 ## Comprehensive Documentation
 
-- [How to use](doc/how-to-use.md)
-- [Configuration](doc/configuration.md)
-- [Development](doc/development.md)
+- [How to use](docs/how-to-use.md)
+- [Configuration](docs/configuration.md)
+- [Development](docs/development.md)
 
 ## Contact Us
 
-For any questions about Multus CNI, feel free to ask a question in #general in the [Intel-Corp Slack](https://intel-corp.herokuapp.com/), or open up a GitHub issue.
+For any questions about Multus CNI, feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/), or open up a GitHub issue. Request an invite to NPWG slack [here](https://intel-corp.herokuapp.com/).

cmd/main.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept refered from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"gopkg.in/intel/multus-cni.v3/pkg/multus"
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+)
+
+
+func main() {
+
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			result, err := multus.CmdAdd(args, nil, nil)
+			if err != nil {
+				return err
+			}
+			return result.Print()
+		},
+		func(args *skel.CmdArgs) error {
+			return multus.CmdCheck(args, nil, nil)
+		},
+		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

deployments/Dockerfile

@@ -10,7 +10,7 @@ RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
     yum install -y $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     cd /usr/src/multus-cni && \
-    ./build
+    ./hack/build-go.sh
 
 FROM centos:centos7
 COPY --from=build /usr/src/multus-cni /usr/src/multus-cni

deployments/Dockerfile.arm64

@@ -8,7 +8,7 @@ ENV GOARCH "arm64"
 ENV GOOS "linux"
 
 RUN  cd /usr/src/multus-cni && \
-     ./build
+     ./hack/build-go.sh
 
 # build arm64 container
 FROM arm64v8/centos:7

deployments/Dockerfile.openshift

@@ -1,11 +1,11 @@
 # This dockerfile is specific to building Multus for OpenShift
-FROM openshift/origin-release:golang-1.10 as builder
+FROM openshift/origin-release:golang-1.15 as builder
 
 ADD . /usr/src/multus-cni
 
 WORKDIR /usr/src/multus-cni
 ENV GO111MODULE=off
-RUN ./build
+RUN ./hack/build-go.sh
 
 FROM openshift/origin-base
 RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin

deployments/Dockerfile.ppc64le

@@ -13,7 +13,7 @@ RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
     yum install -y $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     cd /usr/src/multus-cni && \
-    ./build
+    ./hack/build-go.sh
 
 # build ppc container
 FROM ppc64le/centos:latest

deployments/Dockerfile.s390x

@@ -0,0 +1,19 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.13 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "s390x"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build s390x container
+FROM s390x/python:3-slim
+COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

docs/configuration.md

@@ -40,6 +40,7 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
 * `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
 * `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
 * `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
 * `logFile` (string, optional): file path for log file. multus puts log in given file
 * `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
 * `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
@@ -84,7 +85,15 @@ Only one option is necessary to configure this functionality:
 
 You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration.
 
-Multus will always log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet. This method is always enabled.
+#### Logging via STDERR
+
+By default, Multus will log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet.
+
+Optionally, you may disable this method by setting the `logToStderr` option in your CNI configuration:
+
+```
+    "logToStderr": false,
+```
 
 #### Writing to a Log File
 
@@ -119,6 +128,8 @@ The functionality provided by the `namespaceIsolation` configuration option enab
 
 **NOTE**: The default namespace is special in this scenario. Even with namespace isolation enabled, any pod, in any namespace is allowed to refer to `NetworkAttachmentDefinitions` in the default namespace. This allows you to create commonly used unprivileged `NetworkAttachmentDefinitions` without having to put them in all namespaces. For example, if you had a `NetworkAttachmentDefinition` named `foo` the default namespace, you may reference it in an annotation with: `default/foo`.
 
+**NOTE**: You can also add additional namespaces which can be referred to globally using the `global-namespaces` option (see next section).
+
 For example, if a pod is created in the namespace called `development`, Multus will not allow networks to be attached when defined by custom resources created in a different namespace, say in the `default` network.
 
 Consider the situation where you have a system that has users of different privilege levels -- as an example, a platform which has two administrators: a Senior Administrator and a Junior Administrator. The Senior Administrator may have access to all namespaces, and some network configurations as used by Multus are considered to be privileged in that they allow access to some protected resources available on the network. However, the Junior Administrator has access to only a subset of namespaces, and therefore it should be assumed that the Junior Administrator cannot create pods in their limited subset of namespaces. The `namespaceIsolation` feature provides for this isolation, allowing pods created in given namespaces to only access custom resources in the same namespace as the pod.
@@ -215,7 +226,7 @@ pod/samplepod created
 You'll note that pod fails to spawn successfully. If you check the Multus logs, you'll see an entry such as:
 
 ```
-2018-12-18T21:41:32Z [error] GetPodNetwork: namespace isolation violation: podnamespace: development / target namespace: privileged
+2018-12-18T21:41:32Z [error] GetNetworkDelegates: namespace isolation enabled, annotation violates permission, pod is in namespace development but refers to target namespace privileged
 ```
 
 This error expresses that the pod resides in the namespace named `development` but refers to a `NetworkAttachmentDefinition` outside of that namespace, in this case, the namespace named `privileged`.
@@ -253,6 +264,16 @@ NAME        READY   STATUS    RESTARTS   AGE
 samplepod   1/1     Running   0          31s
 ```
 
+### Allow specific namespaces to be used across namespaces when using namespace isolation
+
+The `globalNamespaces` configuration option is only used when `namespaceIsolation` is set to true. `globalNamespaces` specifies a comma-delimited list of namespaces which can be referred to from outside of any given namespace in which a pod resides.
+
+```
+  "globalNamespaces": "default,namespace-a,namespace-b",
+```
+
+Note that when using `globalNamespaces` the `default` namespace must be specified in the list if you wish to use that namespace, when `globalNamespaces` is not set, the `default` namespace is implied to be used across namespaces.
+
 ### Specify default cluster network in Pod annotations
 
 Users may also specify the default network for any given pod (via annotation), for cases where there are multiple cluster networks available within a Kubernetes cluster.

docs/development.md

@@ -22,12 +22,12 @@ If an issue is closed that you don't feel is sufficiently resolved, please feel
 
 ## How do I build multus-cni?
 
-You can use the built in `./build` script!
+You can use the built in `./hack/build-go.sh` script!
 
 ```
 git clone https://github.com/intel/multus-cni.git
 cd multus-cni
-./build
+./hack/build-go.sh
 ```
 
 ## How do I run CI tests?
@@ -35,7 +35,7 @@ cd multus-cni
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
 ```
-sudo ./test.sh
+sudo ./scripts/test.sh
 ```
 
 ## What are the best practices for logging?

docs/how-to-use.md

@@ -567,7 +567,11 @@ This the directory in which the Multus binary will be installed.
 
     --namespace-isolation=false
 
-Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces.
+Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces. See (the configuration guide for more information)[configuration.md].
+
+    --global-namespaces=default,foo,bar
+
+The `--global-namespaces` works only when `--namespace-isolation=true`. This takes a comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled. See (the configuration guide for more information)[configuration.md].
 
     --multus-bin-file=/usr/src/multus-cni/bin/multus
 

docs/quickstart.md

@@ -29,7 +29,14 @@ To verify that you default network is ready, you may list your Kubernetes nodes
 kubectl get nodes
 ```
 
-In the case that your default network is ready
+In the case that your default network is ready you will see the `STATUS` column also switch to `Ready` for each node.
+
+```
+NAME                  STATUS   ROLES           AGE    VERSION
+master-0              Ready    master          1h     v1.17.1
+master-1              Ready    master          1h     v1.17.1
+master-2              Ready    master          1h     v1.17.1
+```
 
 ## Installation
 

e2e/README.md

@@ -10,3 +10,9 @@ $ ./get_tools.sh
 $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```
+
+### How to teardown cluster
+
+```
+$ ./teardown.sh
+```

e2e/default-route1.yml

@@ -0,0 +1,57 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default-route-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config",
+              "ips": [ "10.1.1.21/24" ] ,
+              "default-route": [ "10.1.1.254" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config", 
+              "ips": [ "10.1.1.22/24" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/get_tools.sh

@@ -5,7 +5,7 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.8.1/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl

e2e/setup_cluster.sh

@@ -3,32 +3,64 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
+kind_network='kind'
 reg_name='kind-registry'
 reg_port='5000'
 running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
 if [ "${running}" != 'true' ]; then
+  # run registry and push the multus image
   docker run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+  docker build -t localhost:5000/multus:e2e ..
+  docker push localhost:5000/multus:e2e
 fi
-reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+reg_host="${reg_name}"
+if [ "${kind_network}" = "bridge" ]; then
+    reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+fi
+echo "Registry Host: ${reg_host}"
 
+# deploy cluster with kind
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_ip}:${reg_port}"]
+    endpoint = ["http://${reg_host}:${reg_port}"]
 nodes:
   - role: control-plane
   - role: worker
   - role: worker
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+containers=$(docker network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
+needs_connect="true"
+for c in $containers; do
+  if [ "$c" = "${reg_name}" ]; then
+    needs_connect="false"
+  fi
+done
+if [ "${needs_connect}" = "true" ]; then
+  docker network connect "${kind_network}" "${reg_name}" || true
+fi
+
 kind export kubeconfig
 sudo env PATH=${PATH} koko -d kind-worker,eth1 -d kind-worker2,eth1
 sleep 1
 kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml
+kubectl create -f multus-daemonset.yml
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
 kubectl create -f cni-install.yml

e2e/setup_multus.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-set -o errexit
-
-export PATH=${PATH}:./bin
-
-kind export kubeconfig
-sudo koko -d kind-worker,eth1 -d kind-worker2,eth1
-sleep 1
-kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml
-sleep 1
-kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
-kubectl create -f cni-install.yml
-sleep 1
-kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/teardown.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+#set -o errexit
+
+reg_name='kind-registry'
+export PATH=${PATH}:./bin
+
+# delete cluster kind
+kind delete cluster
+docker kill ${reg_name}
+docker rm ${reg_name}

e2e/test-default-route1.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f default-route1.yml
+kubectl wait --for=condition=ready -l app=default-route1 --timeout=300s pod
+
+echo "check default-route-worker1 interface: net1"
+kubectl exec default-route-worker1 -- ip a show dev net1
+
+echo "check default-route-worker1 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.21" ]; then
+	echo "default-route-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker1 default route"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.1.1.254" ]; then
+	echo "default-route-worker1 default route is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 interface: net1"
+kubectl exec default-route-worker2 -- ip a show dev net1
+
+echo "check default-route-worker2 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.22" ]; then
+	echo "default-route-worker2 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 default route"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.244.1.1" ]; then
+	echo "default-route-worker2 default route is different: ${ipaddr}"
+fi
+
+echo "cleanup resources"
+kubectl delete -f default-route1.yml

e2e/test-simple-macvlan1.sh

@@ -3,11 +3,11 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f macvlan1.yml
+kubectl create -f simple-macvlan1.yml
 kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
 
 echo "check macvlan1-worker1 interface: net1"
-kubectl exec macvlan1-worker1 ip a show dev net1
+kubectl exec macvlan1-worker1 -- ip a show dev net1
 
 echo "check macvlan1-worker1 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
@@ -17,7 +17,7 @@ if [ $ipaddr != "10.1.1.11" ]; then
 fi
 
 echo "check macvlan1-worker2 interface: net1"
-kubectl exec macvlan1-worker2 ip a show dev net1
+kubectl exec macvlan1-worker2 -- ip a show dev net1
 
 echo "check macvlan1-worker2 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
@@ -25,3 +25,6 @@ ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
 if [ $ipaddr != "10.1.1.12" ]; then
 	echo "macvlan1-worker2 IP address is different: ${ipaddr}"
 fi
+
+echo "cleanup resources"
+kubectl delete -f simple-macvlan1.yml

examples/clusterrole.yml

@@ -1,19 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus-crd-overpowered
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update

examples/cni-configuration.conf

@@ -1,13 +0,0 @@
-{
-  "name": "multus-cni-network",
-  "type": "multus",
-  "delegates": [
-    {
-      "type": "flannel",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  ],
-  "kubeconfig": "/etc/kubernetes/kubelet.conf"
-}

examples/crd.yml

@@ -1,21 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  version: v1
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string

examples/flannel-conf.yml

@@ -1,12 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: flannel-conf
-spec: 
-  config: '{
-    "cniVersion": "0.3.0",
-    "type": "flannel",
-    "delegate": {
-      "isDefaultGateway": true
-    }
-  }'

examples/macvlan-conf.yml

@@ -1,21 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf
-spec: 
-  config: '{
-      "cniVersion": "0.3.0",
-      "type": "macvlan",
-      "master": "eth0",
-      "mode": "bridge",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "192.168.1.0/24",
-        "rangeStart": "192.168.1.200",
-        "rangeEnd": "192.168.1.216",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ],
-        "gateway": "192.168.1.1"
-      }
-    }'

examples/macvlan-pod.yml

@@ -0,0 +1,56 @@
+---
+# This net-attach-def defines macvlan-conf with 
+#   + ips capabilities to specify ip in pod annotation and 
+#   + mac capabilities to specify mac address in pod annotation
+# default gateway is defined as well
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec: 
+  config: '{
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "macvlan",
+          "capabilities": { "ips": true },
+          "master": "eth0",
+          "mode": "bridge",
+          "ipam": {
+            "type": "static",
+            "routes": [
+              {
+                "dst": "0.0.0.0/0",
+                "gw": "10.1.1.1"
+              }
+            ] 
+          }
+        }, {
+          "capabilities": { "mac": true },
+          "type": "tuning"
+        }
+      ]
+    }'
+---
+# Define a pod with macvlan-conf, defined above, with ip address and mac, and 
+# "gateway" overrides default gateway to use macvlan-conf's one. 
+# without "gateway" in k8s.v1.cni.cncf.io/networks, default route will be cluster
+# network interface, eth0, even tough macvlan-conf has default gateway config.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan-conf",
+              "ips": [ "10.1.1.101/24" ],
+              "mac": "c2:b0:57:49:47:f1",
+              "gateway": [ "10.1.1.1" ]
+            }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+    ports:
+    - containerPort: 80

examples/multus-ptp-portmap.conf

@@ -1,36 +0,0 @@
-{
-    "name": "multus-cni-network",
-    "type": "multus"
-    "capabilities": {
-        "portMappings": true
-    },
-    "delegates": [
-        {
-            "cniVersion": "0.3.1",
-            "name": "ptp-tuning-conflist",
-            "plugins": [
-                {
-                    "dns": {
-                        "nameservers": [
-                            "172.16.1.1"
-                        ]
-                    },
-                    "ipMasq": true,
-                    "ipam": {
-                        "subnet": "172.16.0.0/24",
-                        "type": "host-local"
-                    },
-                    "mtu": 512,
-                    "type": "ptp"
-                },
-                {
-                    "capabilities": {
-                        "portMappings": true
-                    },
-                    "externalSetMarkChain": "KUBE-MARK-MASQ",
-                    "type": "portmap"
-                }
-            ]
-        }
-    ],
-}

examples/multus-with-flannel.yml

@@ -1,163 +0,0 @@
-# -----------------------------------------------
-# - Example Configuration Deployment
-# -----------------------------------------------
-# - Deploys a .conf file on each node
-# - Configured for Multus + Flannel.
-# - As well as assets for Flannel
-# - Based on https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml
-# -----------------------------------------------
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel
-subjects:
-- kind: ServiceAccount
-  name: flannel
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: flannel
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-multus-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "delegates": [
-        {
-          "type": "flannel",
-          "delegate": {
-            "isDefaultGateway": true
-          }
-        }
-      ],
-      "kubeconfig": "/etc/kubernetes/kubelet.conf"
-    }
-  net-conf.json: |
-    {
-      "Network": "10.244.0.0/16",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-multus-with-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: multus-cfg
-          configMap:
-            name: kube-multus-cfg

examples/net-resource-sample-pod.yaml

@@ -1,21 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: testpod1
-  labels:
-    env: test
-  annotations:
-    k8s.v1.cni.cncf.io/networks: sriov-net-a
-spec:
-  containers:
-  - name: appcntr1
-    image: centos/tools
-    imagePullPolicy: IfNotPresent
-    command: [ "/bin/bash", "-c", "--" ]
-    args: [ "while true; do sleep 300000; done;" ]
-    resources:
-      requests:
-        intel.com/sriov: '1'
-      limits:
-        intel.com/sriov: '1'
-  restartPolicy: "Never"

examples/npwg-demo-1/01_crd.yml

@@ -1,30 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  # group name to use for REST API: /apis/<group>/<version>
-  group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
-  scope: Namespaced
-  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
-    plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
-    singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
-    kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string
-

examples/npwg-demo-1/02_clusterrole.yml

@@ -1,16 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: multus
-rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'

examples/npwg-demo-1/03_namespace1.yml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: testns1

examples/npwg-demo-1/04_macvlan1.yml

@@ -1,72 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-1
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.101/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-2
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.102/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-3
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.103/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-4
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.104/24" }
-                ]
-            }
-        }'

examples/npwg-demo-1/05_vlan1.yml

@@ -1,19 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: vlan-conf-1-1
-  namespace: testns1
-spec: 
-  config: '{
-                "cniVersion": "0.3.0",
-                "type": "vlan",
-                "master": "eth1",
-                "vlanid": 1,
-                "ipam": {
-                        "type": "static",
-                        "addresses": [
-                        { "address": "172.16.1.101/24"
-                        } ]
-                }
-        }'

examples/npwg-demo-1/06_flannel2.yml

@@ -1,196 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel2
-subjects:
-- kind: ServiceAccount
-  name: flannel2
-  namespace: kube-system
----
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: flannel2
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-flannel2-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-data:
-  flannel2-conf.json: |
-    {
-      "type": "flannel",
-      "name": "flannel-2",
-      "subnetFile": "/run/flannel/flannel2.env",
-      "dataDir": "/var/lib/cni/flannel2",
-      "delegate": {
-        "bridge": "kbr1"
-      }
-    }
-  net-conf.json: |
-    {
-      "Network": "10.144.0.0/16",
-      "SubnetLen": 24,
-      "SubnetMin": "10.144.0.0",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: flannel-etcd
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - etcd
-    - --advertise-client-urls=http://10.1.1.1:12379
-    - --listen-client-urls=http://0.0.0.0:12379
-    - --listen-peer-urls=http://localhost:12380
-    image: quay.io/coreos/etcd:latest
-    name: etcd
-  hostNetwork: true
-  nodeName: kube-master
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: flannel-etcdctl
-  namespace: kube-system
-spec:
-  template:
-    spec:
-      containers:
-      - name: flannel-etcdctl
-        image: quay.io/coreos/etcd:latest
-        command: ["etcdctl"]
-        args: ["--endpoints=http://10.1.1.1:12379", "set", "/flannel2/network/config", '{ "Network": "10.5.0.0/16", "Backend": {"Type": "vxlan", "VNI": 2}}']
-      hostNetwork: true
-      nodeName: kube-master
-      restartPolicy: Never
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-flannel2-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel2
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel2
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/flannel2-conf.json
-        - /etc/cni/multus/net.d/10-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel2
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --etcd-endpoints=http://10.1.1.1:12379
-        - -iface=eth1
-        - -subnet-file=/run/flannel/flannel2.env
-        - -etcd-prefix=/flannel2/network
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          configMap:
-            name: kube-flannel2-cfg
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: flannel-2

examples/npwg-demo-1/07_ptp_tuning_conflist.yml

@@ -1,30 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: ptp-tuning-conflist
-spec: 
-  config: '{
-	"cniVersion": "0.3.1",
-	"name": "ptp-tuning-conflist",
-	"plugins": [{
-			"type": "ptp",
-			"ipMasq": true,
-			"mtu": 512,
-			"ipam": {
-				"type": "host-local",
-				"subnet": "172.16.0.0/24"
-			},
-			"dns": {
-				"nameservers": ["172.16.1.1"]
-			}
-		},
-		{
-			"name": "mytuning",
-			"type": "tuning",
-			"sysctl": {
-				"net.core.somaxconn": "500"
-			}
-		}
-	]
-}'

examples/npwg-demo-1/11_pod_case1.yml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-01
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf-1
-spec:
-  containers:
-  - name: pod-case-01
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/12_pod_case2.yml

@@ -1,18 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-02
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-2" },
-            { "name": "vlan-conf-1-1",
-              "namespace": "testns1",
-              "interface": "vlan1-1" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-02
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/13_pod_case3.yml

@@ -1,17 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-03
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-3" },
-            { "name": "macvlan-conf-4" },
-            { "name": "flannel-2" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-03
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/14_pod_case4.yml

@@ -1,11 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-04
-spec:
-  containers:
-  - name: pod-case-04
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/15_pod_case5.yml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-05
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "ptp-tuning-conflist" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-05
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/sample-pod.yml

@@ -1,14 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
-    image: dougbtv/centos-network
-    ports:
-    - containerPort: 80

examples/sriov-net.yaml

@@ -1,21 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: sriov-net-a
-  annotations:
-    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
-spec:
-  config: '{
-  "type": "sriov",
-  "vlan": 1000,
-  "ipam": {
-    "type": "host-local",
-    "subnet": "10.56.217.0/24",
-    "rangeStart": "10.56.217.171",
-    "rangeEnd": "10.56.217.181",
-    "routes": [{
-      "dst": "0.0.0.0/0"
-    }],
-    "gateway": "10.56.217.1"
-  }
-}'

examples/sriov-pod.yml

@@ -0,0 +1,47 @@
+# This net-attach-def defines SR-IOV CNI config
+# Please see https://github.com/intel/sriov-cni and https://github.com/intel/sriov-network-device-plugin
+# for its detail.
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod1
+  labels:
+    env: test
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a
+spec:
+  containers:
+  - name: appcntr1
+    image: centos/tools
+    imagePullPolicy: IfNotPresent
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "while true; do sleep 300000; done;" ]
+    resources:
+      requests:
+        intel.com/sriov: '1'
+      limits:
+        intel.com/sriov: '1'
+  restartPolicy: "Never"

go.mod

@@ -6,22 +6,21 @@ require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/containernetworking/cni v0.7.1
 	github.com/containernetworking/plugins v0.8.2
-	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200127152046-0ee521d56061
-	github.com/onsi/ginkgo v1.10.1
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d
+	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
 	github.com/pkg/errors v0.8.1
 	github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf
 	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
-	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
+	golang.org/x/net v0.0.0-20201021035429-f5854403a974
 	google.golang.org/grpc v1.23.0
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	k8s.io/api v0.0.0-20181115043458-b799cb063522
-	k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028
-	k8s.io/client-go v0.0.0-20181115111358-9bea17718df8
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	k8s.io/api v0.18.3
+	k8s.io/apimachinery v0.18.3
+	k8s.io/client-go v0.18.3
 	k8s.io/klog v1.0.0
-	k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c // indirect
 	k8s.io/kubernetes v1.13.0
 )
+
+replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2

go.sum

@@ -1,6 +1,15 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -30,11 +39,15 @@ github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjI
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8=
 github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/evanphx/json-patch v4.1.0+incompatible h1:K1MDoo4AZ4wU0GIU/fPmtZg7VpzLjCxu+UwBD1FvwOc=
-github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -46,6 +59,8 @@ github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
@@ -54,48 +69,60 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI=
+github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6tubcjzGMODIVG5hlrCeImaBnZzKF2N8SM=
-github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab h1:k/Biv+LJL35wkk0Hveko1nj7as4tSHkHdZaNlzn/gcQ=
-github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 h1:/UewZcckqhvnnS0C6r3Sher2hSEbVmM6Ogpcjen08+Y=
-github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200127152046-0ee521d56061 h1:zz0mSqgjSJP6gqP2b7GdCiysj5OgD2DMJRNFJegLcs4=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200127152046-0ee521d56061/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d h1:9QbZltQGRFe7temwcTDjj8rIbow48Gv6mIKOxuks+OI=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -108,6 +135,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -117,12 +146,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b h1:Ey6yH0acn50T/v6CB75bGP4EMJqnv9WvnjN7oZaj+xE=
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a h1:KfNOeFvoAssuZLT7IntKZElKwi/5LRuxY71k+t6rfaM=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
@@ -134,17 +164,16 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -158,78 +187,110 @@ github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I=
 github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ=
-golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
+golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3 h1:6KET3Sqa7fkVfD63QnAM81ZeYg5n4HwApOJkufONnHA=
-golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0=
-golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190927073244-c990c680b611 h1:q9u40nxWT5zRClI/uU9dHCiYGottAg6Nzz4YUQyHxdA=
-golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0 h1:7+F62GGWUowoiJOUDivedlBECd/fTeUDJnCu0JetQO0=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw=
-gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
-gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
-google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
@@ -241,46 +302,46 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20181115043458-b799cb063522 h1:JGWVL/WnKVncCXESoe8wCBZJbXbUvYi+AuSHJ/IuFN8=
-k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
-k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028 h1:5i4gnQYlGFIWZp34j0qnt3eeCEFs2ymFzRhzFOPG5/U=
-k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
-k8s.io/client-go v0.0.0-20181115111358-9bea17718df8 h1:Q1pEgTyAmkwHY+lScBj4VZaGP3ykzGUoP9qC0/JxbEk=
-k8s.io/client-go v0.0.0-20181115111358-9bea17718df8/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
-k8s.io/code-generator v0.0.0-20181114232248-ae218e241252 h1:bURaCtXd+lhw5buMNdeUig98d3Eq97Muux1hUFvkqb0=
-k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204=
-k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0=
+k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA=
+k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk=
+k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
+k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k=
+k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
+k8s.io/code-generator v0.18.3 h1:5H57pYEbkMMXCLKD16YQH3yDPAbVLweUsB1M3m70D1c=
+k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab h1:j4L8spMe0tFfBvvW6lrc0c+Ql8+nnkcV3RYfi3eSwGY=
-k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be h1:aWEq4nbj7HRJ0mtKYjNSk/7X28Tl6TI6FeG8gKF+r7Q=
-k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
-k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM=
-k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
-modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
-modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
-modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
-modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU=
+k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

hack/build-go.sh

@@ -19,7 +19,7 @@ if [ -z "$VERSION" ]; then
 	fi
 	set -e
 fi
-DATE=$(date --iso-8601=seconds)
+DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
 COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
 LDFLAGS="-X main.version=${VERSION:-master} -X main.commit=${COMMIT} -X main.date=${DATE}"
 export CGO_ENABLED=0
@@ -40,11 +40,15 @@ if [ "$GO111MODULE" == "off" ]; then
 	export GO15VENDOREXPERIMENT=1
 	export GOBIN=${PWD}/bin
 	export GOPATH=${PWD}/gopath
-	go install -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/multus
+	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
 else
 	# build with go modules
 	export GO111MODULE=on
+	BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+	if [ -n "$MODMODE" ]; then
+		BUILD_ARGS+=(-mod "$MODMODE")
+	fi
 
 	echo "Building plugins"
-	go build -o ${DEST_DIR}/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ./multus
+	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
 fi

images/deprecated/multus-daemonset-crio-pre1.16.yml

@@ -154,7 +154,7 @@ spec:
       containers:
       - name: kube-multus
         # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest
+        image: nfvpe/multus:v3.6
         command: ["/entrypoint.sh"]
         args:
         - "--cni-bin-dir=/host/usr/libexec/cni"

images/deprecated/multus-daemonset-gke-pre-1.16.yml

@@ -134,7 +134,7 @@ spec:
       serviceAccountName: multus
       containers:
         - name: kube-multus
-          image: nfvpe/multus:v3.4
+          image: nfvpe/multus:v3.6
           command: ["/entrypoint.sh"]
           args:
             - "--multus-conf-file=auto"
@@ -229,4 +229,4 @@ spec:
             name: multus-cni-config
             items:
               - key: cni-conf.json
-                path: 70-multus.conf
+                path: 70-multus.conf

images/deprecated/multus-daemonset-pre-1.16.yml

@@ -153,7 +153,7 @@ spec:
       serviceAccountName: multus
       containers:
       - name: kube-multus
-        image: nfvpe/multus:v3.4.1
+        image: nfvpe/multus:v3.6
         command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"

images/entrypoint.sh

@@ -3,6 +3,13 @@
 # Always exit on errors.
 set -e
 
+# Trap sigterm
+function exitonsigterm() {
+  echo "Trapped sigterm, exiting."
+  exit 0
+}
+trap exitonsigterm SIGTERM
+
 # Set our known directories.
 CNI_CONF_DIR="/host/etc/cni/net.d"
 CNI_BIN_DIR="/host/opt/cni/bin"
@@ -12,6 +19,8 @@ MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
 MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
 MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
 MULTUS_NAMESPACE_ISOLATION=false
+MULTUS_GLOBAL_NAMESPACES=""
+MULTUS_LOG_TO_STDERR=true
 MULTUS_LOG_LEVEL=""
 MULTUS_LOG_FILE=""
 MULTUS_READINESS_INDICATOR_FILE=""
@@ -42,7 +51,9 @@ function usage()
     echo -e "\t--skip-multus-binary-copy=$SKIP_BINARY_COPY"
     echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
     echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
+    echo -e "\t--global-namespaces=$MULTUS_GLOBAL_NAMESPACES (used only with --namespace-isolation=true)"
     echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-to-stderr=$MULTUS_LOG_TO_STDERR (empty by default, used only with --multus-conf-file=auto)"
     echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
     echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
     echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
@@ -68,6 +79,10 @@ function warn()
     log "WARN: {$1}"
 }
 
+if ! type python3 &> /dev/null; then
+	alias python=python3
+fi
+
 # Parse parameters given as arguments to this script.
 while [ "$1" != "" ]; do
     PARAM=`echo $1 | awk -F= '{print $1}'`
@@ -98,6 +113,12 @@ while [ "$1" != "" ]; do
         --namespace-isolation)
             MULTUS_NAMESPACE_ISOLATION=$VALUE
             ;;
+        --global-namespaces)
+            MULTUS_GLOBAL_NAMESPACES=$VALUE
+            ;;
+        --multus-log-to-stderr)
+            MULTUS_LOG_TO_STDERR=$VALUE
+            ;;
         --multus-log-level)
             MULTUS_LOG_LEVEL=$VALUE
             ;;
@@ -255,6 +276,17 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
         ISOLATION_STRING="\"namespaceIsolation\": true,"
       fi
 
+      GLOBAL_NAMESPACES_STRING=""
+      if [ ! -z "${MULTUS_GLOBAL_NAMESPACES// }" ]; then
+        GLOBAL_NAMESPACES_STRING="\"globalNamespaces\": \"$MULTUS_GLOBAL_NAMESPACES\","
+      fi
+
+      LOG_TO_STDERR_STRING=""
+      if [ "$MULTUS_LOG_TO_STDERR" == false ]; then
+        LOG_TO_STDERR_STRING="\"logToStderr\": false,"
+      fi
+
+
       LOG_LEVEL_STRING=""
       if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
         case "$MULTUS_LOG_LEVEL" in
@@ -297,11 +329,32 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
 
       if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
         MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python -c 'import json,sys;print json.load(sys.stdin)["name"]')"
+            python -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
       else
         MASTER_PLUGIN_NET_NAME="multus-cni-network"
       fi
 
+      capabilities_python_filter_tmpfile=$(mktemp)
+      cat << EOF > $capabilities_python_filter_tmpfile
+import json,sys
+conf = json.load(sys.stdin)
+capabilities = {}
+if 'plugins' in conf:
+    for capa in [p['capabilities'] for p in conf['plugins'] if 'capabilities' in p]:
+        capabilities.update({capability:enabled for (capability,enabled) in capa.items() if enabled})
+elif 'capabilities' in conf:
+    capabilities.update({capability:enabled for (capability,enabled) in conf['capabilities'] if enabled})
+if len(capabilities) > 0:
+    print("""\"capabilities\": """ + json.dumps(capabilities) + ",")
+else:
+    print("")
+EOF
+
+      NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
+            python $capabilities_python_filter_tmpfile)"
+      rm $capabilities_python_filter_tmpfile
+      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING" 
+
       MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
       MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
       log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
@@ -310,7 +363,10 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
           $CNI_VERSION_STRING
           "name": "$MASTER_PLUGIN_NET_NAME",
           "type": "multus",
+          $NESTED_CAPABILITIES_STRING
           $ISOLATION_STRING
+          $GLOBAL_NAMESPACES_STRING
+          $LOG_TO_STDERR_STRING
           $LOG_LEVEL_STRING
           $LOG_FILE_STRING
           $ADDITIONAL_BIN_DIR_STRING
@@ -374,5 +430,9 @@ if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
   done
 else
   log "Entering sleep (success)..."
-  sleep infinity
+  if tty -s; then
+	  read
+  else
+	  sleep infinity
+  fi
 fi

images/multus-daemonset-crio.yml

@@ -23,6 +23,19 @@ spec:
             networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
           type: object
           properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
             spec:
               description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
               type: object
@@ -159,7 +172,7 @@ spec:
       containers:
       - name: kube-multus
         # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest
+        image: docker.io/nfvpe/multus:stable
         command: ["/entrypoint.sh"]
         args:
         - "--cni-version=0.3.1"
@@ -186,6 +199,7 @@ spec:
           mountPath: /host/usr/libexec/cni
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
       volumes:
         - name: run
           hostPath:
@@ -235,7 +249,7 @@ spec:
       containers:
       - name: kube-multus
         # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
+        image: docker.io/nfvpe/multus:stable-ppc64le
         command: ["/entrypoint.sh"]
         args:
         - "--cni-version=0.3.1"
@@ -258,6 +272,147 @@ spec:
           mountPath: /host/usr/libexec/cni
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-arm64v8
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: docker.io/nfvpe/multus:stable-arm64v8
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-version=0.3.1"
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-s390x
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: s390x
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: docker.io/nfvpe/multus:stable-s390x
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-version=0.3.1"
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
       volumes:
         - name: cni
           hostPath:

images/multus-daemonset-gke-1.16.yml

@@ -144,7 +144,7 @@ spec:
       serviceAccountName: multus
       containers:
         - name: kube-multus
-          image: nfvpe/multus:v3.4
+          image: docker.io/nfvpe/multus:stable
           command: ["/entrypoint.sh"]
           args:
             - "--multus-conf-file=auto"
@@ -212,7 +212,7 @@ spec:
       containers:
         - name: kube-multus
           # ppc64le support requires multus:latest for now. support 3.3 or later.
-          image: nfvpe/multus:latest-ppc64le
+          image: docker.io/nfvpe/multus:stable-ppc64le
           command: ["/entrypoint.sh"]
           args:
             - "--multus-conf-file=auto"
@@ -246,4 +246,4 @@ spec:
             name: multus-cni-config
             items:
               - key: cni-conf.json
-                path: 70-multus.conf
+                path: 70-multus.conf

images/multus-daemonset.yml

@@ -23,6 +23,19 @@ spec:
             networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
           type: object
           properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
             spec:
               description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
               type: object
@@ -158,7 +171,7 @@ spec:
       serviceAccountName: multus
       containers:
       - name: kube-multus
-        image: nfvpe/multus:v3.4.1
+        image: docker.io/nfvpe/multus:stable
         command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"
@@ -179,6 +192,7 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
       volumes:
         - name: cni
           hostPath:
@@ -225,7 +239,7 @@ spec:
       containers:
       - name: kube-multus
         # ppc64le support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
+        image: docker.io/nfvpe/multus:stable-ppc64le
         command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"
@@ -246,6 +260,141 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-arm64v8
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: docker.io/nfvpe/multus:stable-arm64v8
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-s390x
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: s390x
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: docker.io/nfvpe/multus:stable-s390x
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
       volumes:
         - name: cni
           hostPath:

pkg/checkpoint/checkpoint.go

@@ -19,8 +19,8 @@ import (
 	"encoding/json"
 	"io/ioutil"
 
-	"gopkg.in/intel/multus-cni.v3/logging"
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 )
 

pkg/checkpoint/checkpoint_test.go

@@ -10,7 +10,7 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"

pkg/k8sclient/k8sclient.go

@@ -15,6 +15,7 @@
 package k8sclient
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net"
@@ -37,9 +38,9 @@ import (
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
-	"gopkg.in/intel/multus-cni.v3/kubeletclient"
-	"gopkg.in/intel/multus-cni.v3/logging"
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/kubeletclient"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
@@ -66,22 +67,22 @@ type ClientInfo struct {
 
 // AddPod adds pod into kubernetes
 func (c *ClientInfo) AddPod(pod *v1.Pod) (*v1.Pod, error) {
-	return c.Client.Core().Pods(pod.ObjectMeta.Namespace).Create(pod)
+	return c.Client.CoreV1().Pods(pod.ObjectMeta.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
 }
 
 // GetPod gets pod from kubernetes
 func (c *ClientInfo) GetPod(namespace, name string) (*v1.Pod, error) {
-	return c.Client.Core().Pods(namespace).Get(name, metav1.GetOptions{})
+	return c.Client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
 // DeletePod deletes a pod from kubernetes
 func (c *ClientInfo) DeletePod(namespace, name string) error {
-	return c.Client.Core().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
+	return c.Client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
 }
 
 // AddNetAttachDef adds net-attach-def into kubernetes
 func (c *ClientInfo) AddNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) (*nettypes.NetworkAttachmentDefinition, error) {
-	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(netattach)
+	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
 }
 
 // Eventf puts event into kubernetes events
@@ -211,6 +212,12 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 				return nil, logging.Errorf("parsePodNetworkAnnotation: failed to mac: %v", err)
 			}
 		}
+		if n.InfinibandGUIDRequest != "" {
+			// validate GUID address
+			if _, err := net.ParseMAC(n.InfinibandGUIDRequest); err != nil {
+				return nil, logging.Errorf("parsePodNetworkAnnotation: failed to validate infiniband GUID: %v", err)
+			}
+		}
 		if n.IPRequest != nil {
 			for _, ip := range n.IPRequest {
 				// validate IP address
@@ -235,7 +242,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 
 	logging.Debugf("getKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
-	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(net.Name, metav1.GetOptions{})
+	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(context.TODO(), net.Name, metav1.GetOptions{})
 	if err != nil {
 		errMsg := fmt.Sprintf("cannot find a network-attachment-definition (%s) in namespace (%s): %v", net.Name, net.Namespace, err)
 		if client != nil {
@@ -278,7 +285,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		return nil, resourceMap, err
 	}
 
-	delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID)
+	delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID, resourceName)
 	if err != nil {
 		return nil, resourceMap, err
 	}
@@ -301,33 +308,26 @@ func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 
 // TryLoadPodDelegates attempts to load Kubernetes-defined delegates and add them to the Multus config.
 // Returns the number of Kubernetes-defined delegates added or an error.
-func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, clientInfo *ClientInfo) (int, *v1.Pod, *ClientInfo, error) {
+func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInfo, resourceMap map[string]*types.ResourceInfo) (int, *ClientInfo, error) {
 	var err error
 
-	logging.Debugf("TryLoadPodDelegates: %v, %v, %v", k8sArgs, conf, clientInfo)
+	logging.Debugf("TryLoadPodDelegates: %v, %v, %v", pod, conf, clientInfo)
 	clientInfo, err = GetK8sClient(conf.Kubeconfig, clientInfo)
 	if err != nil {
-		return 0, nil, nil, err
+		return 0, nil, err
 	}
 
 	if clientInfo == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
-			return 0, nil, nil, logging.Errorf("TryLoadPodDelegates: must have either Kubernetes config or delegates")
+			return 0, nil, logging.Errorf("TryLoadPodDelegates: must have either Kubernetes config or delegates")
 		}
-		return 0, nil, nil, nil
-	}
-
-	// Get the pod info. If cannot get it, we use cached delegates
-	pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-	if err != nil {
-		logging.Debugf("TryLoadPodDelegates: Err in loading K8s cluster default network from pod annotation: %v, use cached delegates", err)
-		return 0, nil, nil, nil
+		return 0, nil, nil
 	}
 
 	delegate, err := tryLoadK8sPodDefaultNetwork(clientInfo, pod, conf)
 	if err != nil {
-		return 0, nil, nil, logging.Errorf("TryLoadPodDelegates: error in loading K8s cluster default network from pod annotation: %v", err)
+		return 0, nil, logging.Errorf("TryLoadPodDelegates: error in loading K8s cluster default network from pod annotation: %v", err)
 	}
 	if delegate != nil {
 		logging.Debugf("TryLoadPodDelegates: Overwrite the cluster default network with %v from pod annotations", delegate)
@@ -337,17 +337,17 @@ func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, clientInfo
 
 	networks, err := GetPodNetwork(pod)
 	if networks != nil {
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, conf.ConfDir, conf.NamespaceIsolation)
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, conf, resourceMap)
 
 		if err != nil {
 			if _, ok := err.(*NoK8sNetworkError); ok {
-				return 0, nil, clientInfo, nil
+				return 0, clientInfo, nil
 			}
-			return 0, nil, nil, logging.Errorf("TryLoadPodDelegates: error in getting k8s network for pod: %v", err)
+			return 0, nil, logging.Errorf("TryLoadPodDelegates: error in getting k8s network for pod: %v", err)
 		}
 
 		if err = conf.AddDelegates(delegates); err != nil {
-			return 0, nil, nil, err
+			return 0, nil, err
 		}
 
 		// Check gatewayRequest is configured in delegates
@@ -364,10 +364,10 @@ func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, clientInfo
 			types.CheckGatewayConfig(conf.Delegates)
 		}
 
-		return len(delegates), pod, clientInfo, nil
+		return len(delegates), clientInfo, nil
 	}
 
-	return 0, pod, clientInfo, nil
+	return 0, clientInfo, nil
 }
 
 // GetK8sClient gets client info from kubeconfig
@@ -449,11 +449,8 @@ func GetPodNetwork(pod *v1.Pod) ([]*types.NetworkSelectionElement, error) {
 }
 
 // GetNetworkDelegates returns delegatenetconf from net-attach-def annotation in pod
-func GetNetworkDelegates(k8sclient *ClientInfo, pod *v1.Pod, networks []*types.NetworkSelectionElement, confdir string, confnamespaceIsolation bool) ([]*types.DelegateNetConf, error) {
-	logging.Debugf("GetNetworkDelegates: %v, %v, %v, %v, %v", k8sclient, pod, networks, confdir, confnamespaceIsolation)
-	// resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation.
-	// This will only be initialized once and all delegate objects can reference this to look up device info.
-	var resourceMap map[string]*types.ResourceInfo
+func GetNetworkDelegates(k8sclient *ClientInfo, pod *v1.Pod, networks []*types.NetworkSelectionElement, conf *types.NetConf, resourceMap map[string]*types.ResourceInfo) ([]*types.DelegateNetConf, error) {
+	logging.Debugf("GetNetworkDelegates: %v, %v, %v, %v, %v", k8sclient, pod, networks, conf, resourceMap)
 
 	// Read all network objects referenced by 'networks'
 	var delegates []*types.DelegateNetConf
@@ -463,16 +460,16 @@ func GetNetworkDelegates(k8sclient *ClientInfo, pod *v1.Pod, networks []*types.N
 
 		// The pods namespace (stored as defaultNamespace, does not equal the annotation's target namespace in net.Namespace)
 		// In the case that this is a mismatch when namespaceisolation is enabled, this should be an error.
-		if confnamespaceIsolation {
+		if conf.NamespaceIsolation {
 			if defaultNamespace != net.Namespace {
-				// There is an exception however, we always allow a reference to the default namespace.
-				if net.Namespace != "default" {
+				// We allow exceptions based on the specified list of non-isolated namespaces (and/or "default" namespace, by default)
+				if !isValidNamespaceReference(net.Namespace, conf.NonIsolatedNamespaces) {
 					return nil, logging.Errorf("GetNetworkDelegates: namespace isolation enabled, annotation violates permission, pod is in namespace %v but refers to target namespace %v", defaultNamespace, net.Namespace)
 				}
 			}
 		}
 
-		delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, pod, resourceMap)
+		delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, conf.ConfDir, pod, resourceMap)
 		if err != nil {
 			return nil, logging.Errorf("GetNetworkDelegates: failed getting the delegate: %v", err)
 		}
@@ -483,43 +480,36 @@ func GetNetworkDelegates(k8sclient *ClientInfo, pod *v1.Pod, networks []*types.N
 	return delegates, nil
 }
 
-func getDefaultNetDelegateCRD(client *ClientInfo, net, confdir, namespace string) (*types.DelegateNetConf, error) {
-	logging.Debugf("getDefaultNetDelegateCRD: %v, %v, %s, %s", client, net, confdir, namespace)
-	customResource, err := client.NetClient.NetworkAttachmentDefinitions(namespace).Get(net, metav1.GetOptions{})
-	if err != nil {
-		return nil, logging.Errorf("getDefaultNetDelegateCRD: failed to get network resource: %v", err)
+func isValidNamespaceReference(targetns string, allowednamespaces []string) bool {
+	for _, eachns := range allowednamespaces {
+		if eachns == targetns {
+			return true
+		}
 	}
-
-	configBytes, err := netutils.GetCNIConfig(customResource, confdir)
-	if err != nil {
-		return nil, err
-	}
-
-	delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
-	if err != nil {
-		return nil, err
-	}
-
-	return delegate, nil
+	return false
 }
 
-func getNetDelegate(client *ClientInfo, netname, confdir, namespace string) (*types.DelegateNetConf, error) {
+func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 	logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace)
 	// option1) search CRD object for the network
-	delegate, err := getDefaultNetDelegateCRD(client, netname, confdir, namespace)
+	net := &types.NetworkSelectionElement{
+		Name:      netname,
+		Namespace: namespace,
+	}
+	delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
 	if err == nil {
-		return delegate, nil
+		return delegate, resourceMap, nil
 	}
 
 	// option2) search CNI json config file
 	var configBytes []byte
 	configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
 	if err == nil {
-		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
 		if err != nil {
-			return nil, err
+			return nil, resourceMap, err
 		}
-		return delegate, nil
+		return delegate, resourceMap, nil
 	}
 
 	// option3) search directry
@@ -528,65 +518,66 @@ func getNetDelegate(client *ClientInfo, netname, confdir, namespace string) (*ty
 		if fInfo.IsDir() {
 			files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
 			if err != nil {
-				return nil, err
+				return nil, resourceMap, err
 			}
 			if len(files) > 0 {
 				var configBytes []byte
 				configBytes, err = netutils.GetCNIConfigFromFile("", netname)
 				if err == nil {
-					delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+					delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
 					if err != nil {
-						return nil, err
+						return nil, resourceMap, err
 					}
-					return delegate, nil
+					return delegate, resourceMap, nil
 				}
-				return nil, err
+				return nil, resourceMap, err
 			}
 		}
 	}
-	return nil, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
+	return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
 }
 
 // GetDefaultNetworks parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
-func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient *ClientInfo) error {
-	logging.Debugf("GetDefaultNetworks: %v, %v, %v", k8sArgs, conf, kubeClient)
+func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo, resourceMap map[string]*types.ResourceInfo) (map[string]*types.ResourceInfo, error) {
+	logging.Debugf("GetDefaultNetworks: %v, %v, %v, %v", pod, conf, kubeClient, resourceMap)
 	var delegates []*types.DelegateNetConf
 
 	kubeClient, err := GetK8sClient(conf.Kubeconfig, kubeClient)
 	if err != nil {
-		return err
+		return resourceMap, err
 	}
 	if kubeClient == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
-			return logging.Errorf("GetDefaultNetworks: must have either Kubernetes config or delegates")
+			return resourceMap, logging.Errorf("GetDefaultNetworks: must have either Kubernetes config or delegates")
 		}
-		return nil
+		return resourceMap, nil
 	}
 
-	delegate, err := getNetDelegate(kubeClient, conf.ClusterNetwork, conf.ConfDir, conf.MultusNamespace)
+	delegate, resourceMap, err := getNetDelegate(kubeClient, pod, conf.ClusterNetwork, conf.ConfDir, conf.MultusNamespace, resourceMap)
+
 	if err != nil {
-		return err
+		return resourceMap, logging.Errorf("GetDefaultNetworks: failed to get clusterNetwork %s in namespace %s", conf.ClusterNetwork, conf.MultusNamespace)
 	}
 	delegate.MasterPlugin = true
 	delegates = append(delegates, delegate)
 
 	// Pod in kube-system namespace does not have default network for now.
-	if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), conf.SystemNamespaces) {
+	if !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
 		for _, netname := range conf.DefaultNetworks {
-			delegate, err := getNetDelegate(kubeClient, netname, conf.ConfDir, conf.MultusNamespace)
+			delegate, resourceMap, err := getNetDelegate(kubeClient, pod, netname, conf.ConfDir, conf.MultusNamespace, resourceMap)
 			if err != nil {
-				return err
+				return resourceMap, err
 			}
 			delegates = append(delegates, delegate)
 		}
 	}
 
 	if err = conf.AddDelegates(delegates); err != nil {
-		return err
+		return resourceMap, err
 	}
 
-	return nil
+	return resourceMap, nil
 }
 
 // tryLoadK8sPodDefaultNetwork get pod default network from annotations

pkg/k8sclient/k8sclient_test.go

@@ -23,10 +23,10 @@ import (
 	"testing"
 
 	types020 "github.com/containernetworking/cni/pkg/types/020"
-	testutils "gopkg.in/intel/multus-cni.v3/testing"
+	testutils "gopkg.in/intel/multus-cni.v3/pkg/testing"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
@@ -54,10 +54,21 @@ func NewFakeClientInfo() *ClientInfo {
 var _ = Describe("k8sclient operations", func() {
 	var tmpDir string
 	var err error
+	var genericConf string
 
 	BeforeEach(func() {
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
 		Expect(err).NotTo(HaveOccurred())
+		genericConf = `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"delegates": [{
+			"name": "weave1",
+				"cniVersion": "0.2.0",
+				"type": "weave-net"
+			}],
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
 	})
 
 	AfterEach(func() {
@@ -104,7 +115,9 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(delegates)).To(Equal(2))
@@ -139,7 +152,9 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(len(delegates)).To(Equal(0))
 		Expect(err).To(MatchError("GetNetworkDelegates: failed getting the delegate: getKubernetesDelegate: cannot find a network-attachment-definition (net1) in namespace (test): network-attachment-definitions.k8s.cni.cncf.io \"net1\" not found"))
 	})
@@ -188,7 +203,9 @@ var _ = Describe("k8sclient operations", func() {
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(delegates)).To(Equal(3))
@@ -262,7 +279,9 @@ var _ = Describe("k8sclient operations", func() {
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(delegates)).To(Equal(3))
@@ -306,7 +325,9 @@ var _ = Describe("k8sclient operations", func() {
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(delegates)).To(Equal(2))
@@ -333,7 +354,9 @@ var _ = Describe("k8sclient operations", func() {
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(delegates)).To(Equal(1))
@@ -368,7 +391,9 @@ var _ = Describe("k8sclient operations", func() {
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, tmpDir, false)
+		netConf, err := types.LoadNetConf([]byte(genericConf))
+		netConf.ConfDir = tmpDir
+		delegates, err := GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
 		Expect(len(delegates)).To(Equal(0))
 		Expect(err).To(MatchError(fmt.Sprintf("GetNetworkDelegates: failed getting the delegate: GetCNIConfig: err in GetCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name)))
 	})
@@ -395,10 +420,10 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "myCRD1", "{\"type\": \"mynet\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
@@ -431,10 +456,10 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "myCRD2", "{\"type\": \"mynet2\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(2))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
@@ -471,10 +496,10 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "myCRD2", "{\"type\": \"mynet2\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
@@ -510,10 +535,10 @@ var _ = Describe("k8sclient operations", func() {
 			}`))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myFile1"))
@@ -546,10 +571,10 @@ var _ = Describe("k8sclient operations", func() {
 				"cniVersion": "0.2.0"
 			}`))
 		Expect(err).NotTo(HaveOccurred())
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
@@ -575,10 +600,10 @@ var _ = Describe("k8sclient operations", func() {
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -608,18 +633,17 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "net2", "{\"type\": \"mynet2\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2"))
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 
-		numK8sDelegates, pod, _, err := TryLoadPodDelegates(k8sArgs, netConf, clientInfo)
+		numK8sDelegates, _, err := TryLoadPodDelegates(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(pod).NotTo(BeNil())
 		Expect(numK8sDelegates).To(Equal(0))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1"))
@@ -649,14 +673,14 @@ var _ = Describe("k8sclient operations", func() {
 		_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net2", ""))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 		Expect(err).To(HaveOccurred())
 
 		netConf.ConfDir = "badfilepath"
-		_, _, _, err = TryLoadPodDelegates(k8sArgs, netConf, clientInfo)
+		_, _, err = TryLoadPodDelegates(fakePod, netConf, clientInfo, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -688,10 +712,10 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "net1", "{\"type\": \"mynet1\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		numK8sDelegates, _, _, err := TryLoadPodDelegates(k8sArgs, netConf, clientInfo)
+		numK8sDelegates, _, err := TryLoadPodDelegates(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(numK8sDelegates).To(Equal(0))
 		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
@@ -725,10 +749,10 @@ var _ = Describe("k8sclient operations", func() {
 			testutils.NewFakeNetAttachDef("kube-system", "net1", "{\"type\": \"mynet1\"}"))
 		Expect(err).NotTo(HaveOccurred())
 
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		_, _, _, err = TryLoadPodDelegates(k8sArgs, netConf, nil)
+		_, _, err = TryLoadPodDelegates(fakePod, netConf, nil, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -758,15 +782,15 @@ var _ = Describe("k8sclient operations", func() {
 		_, err = clientInfo.AddNetAttachDef(
 			testutils.NewFakeNetAttachDef("kube-system", "net1", "{\"type\": \"mynet1\"}"))
 		Expect(err).NotTo(HaveOccurred())
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		_, _, _, err = TryLoadPodDelegates(k8sArgs, netConf, nil)
+		_, _, err = TryLoadPodDelegates(fakePod, netConf, nil, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		// additionally, we expect the test to fail with no delegates, as at least one is always required.
 		netConf.Delegates = nil
-		_, _, _, err = TryLoadPodDelegates(k8sArgs, netConf, nil)
+		_, _, err = TryLoadPodDelegates(fakePod, netConf, nil, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -822,10 +846,10 @@ users:
 		_, err = clientInfo.AddNetAttachDef(
 			testutils.NewFakeNetAttachDef("kube-system", "net1", "{\"type\": \"mynet1\"}"))
 		Expect(err).NotTo(HaveOccurred())
-		k8sArgs, err := GetK8sArgs(args)
+		_, err = GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
 
-		_, _, _, err = TryLoadPodDelegates(k8sArgs, netConf, nil)
+		_, _, err = TryLoadPodDelegates(fakePod, netConf, clientInfo, nil)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -843,7 +867,6 @@ users:
 			"namespaceIsolation": true
 		}`
 
-		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
 		net1 := `{
@@ -868,12 +891,64 @@ users:
 		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 		networks, err := GetPodNetwork(pod)
 		Expect(err).NotTo(HaveOccurred())
-		_, err = GetNetworkDelegates(clientInfo, pod, networks, tmpDir, netConf.NamespaceIsolation)
+
+		netConf, err := types.LoadNetConf([]byte(conf))
+		netConf.ConfDir = tmpDir
+		_, err = GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
+
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError("GetNetworkDelegates: namespace isolation enabled, annotation violates permission, pod is in namespace test but refers to target namespace kube-system"))
 
 	})
 
+	It("Properly allows a specified namespace reference when namespace isolation is enabled", func() {
+		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"delegates": [{
+			"name": "weave1",
+				"cniVersion": "0.2.0",
+				"type": "weave-net"
+			}],
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml",
+			"namespaceIsolation": true,
+			"globalNamespaces": "kube-system,donkey-kong"
+		}`
+
+		Expect(err).NotTo(HaveOccurred())
+
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		clientInfo := NewFakeClientInfo()
+		_, err = clientInfo.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		netConf, err := types.LoadNetConf([]byte(conf))
+		netConf.ConfDir = tmpDir
+		_, err = GetNetworkDelegates(clientInfo, pod, networks, netConf, nil)
+
+		Expect(err).NotTo(HaveOccurred())
+
+	})
+
 	Context("Error function", func() {
 		It("Returns proper error message", func() {
 			err := &NoK8sNetworkError{"no kubernetes network found"}
@@ -906,11 +981,11 @@ users:
 			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net2", ""))
 			Expect(err).NotTo(HaveOccurred())
 
-			k8sArgs, err := GetK8sArgs(args)
+			_, err = GetK8sArgs(args)
 			Expect(err).NotTo(HaveOccurred())
 
 			netConf.ConfDir = "garbage value"
-			err = GetDefaultNetworks(k8sArgs, netConf, clientInfo)
+			_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -967,7 +1042,9 @@ users:
 			networks, err := GetPodNetwork(fakePod)
 			Expect(err).NotTo(HaveOccurred())
 
-			_, err = GetNetworkDelegates(clientInfo, fakePod, networks, tmpDir, false)
+			netConf, err := types.LoadNetConf([]byte(genericConf))
+			netConf.ConfDir = tmpDir
+			_, err = GetNetworkDelegates(clientInfo, fakePod, networks, netConf, nil)
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -1185,10 +1262,10 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1267,10 +1344,10 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1327,10 +1404,10 @@ users:
 		}`
 			// note that the provided kubeconfig is invalid
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1386,10 +1463,10 @@ users:
 			}
 		}`
 
-			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 

pkg/kubeletclient/kubeletclient.go

@@ -5,9 +5,9 @@ import (
 	"path/filepath"
 	"time"
 
-	"gopkg.in/intel/multus-cni.v3/checkpoint"
-	"gopkg.in/intel/multus-cni.v3/logging"
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/checkpoint"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	"golang.org/x/net/context"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"

pkg/kubeletclient/kubeletclient_test.go

@@ -15,7 +15,7 @@ import (
 	k8sTypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 
-	mtypes "gopkg.in/intel/multus-cni.v3/types"
+	mtypes "gopkg.in/intel/multus-cni.v3/pkg/types"
 	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
 )
 
@@ -91,10 +91,8 @@ func tearDown(path string) error {
 	if fakeServer != nil {
 		fakeServer.server.Stop()
 	}
-	if err := os.RemoveAll(path); err != nil {
-		return err
-	}
-	return nil
+	err := os.RemoveAll(path)
+	return err
 }
 
 var _ = BeforeSuite(func() {

pkg/logging/logging.go

@@ -16,11 +16,13 @@ package logging
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
+	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
 
 // Level type
@@ -37,7 +39,7 @@ const (
 )
 
 var loggingStderr bool
-var loggingFp *os.File
+var loggingW io.Writer
 var loggingLevel Level
 
 const defaultTimestampFormat = time.RFC3339
@@ -69,10 +71,10 @@ func printf(level Level, format string, a ...interface{}) {
 		fmt.Fprintf(os.Stderr, "\n")
 	}
 
-	if loggingFp != nil {
-		fmt.Fprintf(loggingFp, header, t.Format(defaultTimestampFormat), level)
-		fmt.Fprintf(loggingFp, format, a...)
-		fmt.Fprintf(loggingFp, "\n")
+	if loggingW != nil {
+		fmt.Fprintf(loggingW, header, t.Format(defaultTimestampFormat), level)
+		fmt.Fprintf(loggingW, format, a...)
+		fmt.Fprintf(loggingW, "\n")
 	}
 }
 
@@ -139,16 +141,18 @@ func SetLogFile(filename string) {
 		return
 	}
 
-	fp, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
-	if err != nil {
-		loggingFp = nil
-		fmt.Fprintf(os.Stderr, "multus logging: cannot open %s", filename)
+	loggingW = &lumberjack.Logger{
+		Filename:   filename,
+		MaxSize:    100, // megabytes
+		MaxBackups: 5,
+		MaxAge:     5, // days
+		Compress:   true,
 	}
-	loggingFp = fp
+
 }
 
 func init() {
 	loggingStderr = true
-	loggingFp = nil
+	loggingW = nil
 	loggingLevel = PanicLevel
 }

pkg/logging/logging_test.go

@@ -30,24 +30,24 @@ var _ = Describe("logging operations", func() {
 
 	BeforeEach(func() {
 		loggingStderr = false
-		loggingFp = nil
+		loggingW = nil
 		loggingLevel = PanicLevel
 	})
 
 	It("Check file setter with empty", func() {
 		SetLogFile("")
-		Expect(loggingFp).To(BeNil())
+		Expect(loggingW).To(BeNil())
 	})
 
 	It("Check file setter with empty", func() {
 		SetLogFile("/tmp/foobar.logging")
-		Expect(loggingFp).NotTo(Equal(nil))
+		Expect(loggingW).NotTo(Equal(nil))
 		// check file existance
 	})
 
 	It("Check file setter with bad filepath", func() {
 		SetLogFile("/invalid/filepath")
-		Expect(loggingFp).NotTo(Equal(nil))
+		Expect(loggingW).NotTo(Equal(nil))
 		// check file existance
 	})
 

pkg/multus/multus.go

@@ -12,11 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This is a "Multi-plugin".The delegate concept refered from CNI project
-// It reads other plugin netconf, and then invoke them, e.g.
-// flannel or sriov plugin.
-
-package main
+package multus
 
 import (
 	"context"
@@ -37,25 +33,31 @@ import (
 	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
 	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
-	k8s "gopkg.in/intel/multus-cni.v3/k8sclient"
-	"gopkg.in/intel/multus-cni.v3/logging"
-	"gopkg.in/intel/multus-cni.v3/netutils"
-	"gopkg.in/intel/multus-cni.v3/types"
+	k8s "gopkg.in/intel/multus-cni.v3/pkg/k8sclient"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	"gopkg.in/intel/multus-cni.v3/pkg/netutils"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/vishvananda/netlink"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-var version = "master@git"
-var commit = "unknown commit"
-var date = "unknown date"
+var (
+	version = "master@git"
+	commit  = "unknown commit"
+	date    = "unknown date"
+)
 
-var pollDuration = 1000 * time.Millisecond
-var pollTimeout = 45 * time.Second
+var (
+	pollDuration = 1000 * time.Millisecond
+	pollTimeout  = 45 * time.Second
+)
 
-func printVersionString() string {
+//PrintVersionString ...
+func PrintVersionString() string {
 	return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s",
 		version, commit, date)
 }
@@ -99,6 +101,20 @@ func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string {
 	return fmt.Sprintf("net%d", idx)
 }
 
+func getDelegateDeviceInfo(delegate *types.DelegateNetConf, runtimeConf *libcni.RuntimeConf) (*nettypes.DeviceInfo, error) {
+	// If the DPDeviceInfoFile was created, it was copied to the CNIDeviceInfoFile.
+	// If the DPDeviceInfoFile was not created, CNI might have created it. So
+	// either way, load CNIDeviceInfoFile.
+	if info, ok := runtimeConf.CapabilityArgs["CNIDeviceInfoFile"]; ok {
+		if infostr, ok := info.(string); ok {
+			return nadutils.LoadDeviceInfoFromCNI(infostr)
+		}
+	} else {
+		logging.Debugf("getDelegateDeviceInfo(): No CapArgs - info=%v ok=%v", info, ok)
+	}
+	return nil, nil
+}
+
 func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetConf) error {
 	logging.Debugf("saveDelegates: %s, %s, %v", containerID, dataDir, delegates)
 	delegatesBytes, err := json.Marshal(delegates)
@@ -438,20 +454,31 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 	return err
 }
 
-func delPlugins(exec invoke.Exec, pod *v1.Pod, argIfname string, delegates []*types.DelegateNetConf, lastIdx int, rt *libcni.RuntimeConf, binDir string) error {
-	logging.Debugf("delPlugins: %v, %v, %s, %v, %d, %v, %s", exec, pod, argIfname, delegates, lastIdx, rt, binDir)
+// delPlugins deletes plugins in reverse order from lastdIdx
+// Uses netRt as base RuntimeConf (coming from NetConf) but merges it
+// with each of the delegates' configuration
+func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, binDir string) error {
+	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v, %s", exec, pod, args, k8sArgs, delegates, lastIdx, netRt, binDir)
 	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return logging.Errorf("delPlugins: error setting envionment variable CNI_COMMAND to a value of DEL")
+		return logging.Errorf("delPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
 	}
 
 	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
-		ifName := getIfname(delegates[idx], argIfname, idx)
-		rt.IfName = ifName
+		ifName := getIfname(delegates[idx], args.IfName, idx)
+		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
 		if err := delegateDel(exec, pod, ifName, delegates[idx], rt, binDir); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
+		if cniDeviceInfoPath != "" {
+			err := nadutils.CleanDeviceInfoForCNI(cniDeviceInfoPath)
+			// Even if the filename is set, file may not be present. Ignore error,
+			// but log and in the future may need to filter on specific errors.
+			if err != nil {
+				logging.Debugf("delPlugins: CleanDeviceInfoForCNI returned an error - err=%v", err)
+			}
+		}
 	}
 
 	// Check if we had any errors, and send them all back.
@@ -478,9 +505,10 @@ func cmdPluginErr(k8sArgs *types.K8sArgs, confName string, format string, args .
 	return logging.Errorf(msg+format, args...)
 }
 
-func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) {
+//CmdAdd ...
+func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) {
 	n, err := types.LoadNetConf(args.StdinData)
-	logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient)
+	logging.Debugf("CmdAdd: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
 		return nil, cmdErr(nil, "error loading netconf: %v", err)
 	}
@@ -505,8 +533,36 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 	}
 
+	pod := (*v1.Pod)(nil)
+	if kubeClient != nil {
+		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		if err != nil {
+			var waitErr error
+			// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval
+			if errors.IsServiceUnavailable(err) {
+				pollDuration := 500 * time.Millisecond
+				pollTimeout := 5 * time.Second
+				waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
+					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+					return pod != nil, err
+				})
+				// retry failed, then return error with retry out
+				if waitErr != nil {
+					return nil, cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err)
+				}
+			} else {
+				// Other case, return error
+				return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
+			}
+		}
+	}
+
+	// resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation.
+	// This will only be initialized once and all delegate objects can reference this to look up device info.
+	var resourceMap map[string]*types.ResourceInfo
+
 	if n.ClusterNetwork != "" {
-		err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient)
+		resourceMap, err = k8s.GetDefaultNetworks(pod, n, kubeClient, resourceMap)
 		if err != nil {
 			return nil, cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err)
 		}
@@ -514,7 +570,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		n.Delegates[0].MasterPlugin = true
 	}
 
-	_, pod, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient)
+	_, kc, err := k8s.TryLoadPodDelegates(pod, n, kubeClient, resourceMap)
 	if err != nil {
 		return nil, cmdErr(k8sArgs, "error loading k8s delegates k8s args: %v", err)
 	}
@@ -529,9 +585,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
+		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
+		if cniDeviceInfoPath != "" {
+			err = nadutils.CopyDeviceInfoForCNIFromDP(cniDeviceInfoPath, delegate.ResourceName, delegate.DeviceID)
+			// Even if the filename is set, file may not be present. Ignore error,
+			// but log and in the future may need to filter on specific errors.
+			if err != nil {
+				logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
+			}
+		}
 
-		runtimeConfig := types.MergeCNIRuntimeConfig(n.RuntimeConfig, delegate)
-		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, runtimeConfig)
 		tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n.BinDir, cniArgs)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
@@ -540,7 +603,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 				netName = delegate.ConfList.Name
 			}
 			// Ignore errors; DEL must be idempotent anyway
-			_ = delPlugins(exec, nil, args.IfName, n.Delegates, idx, rt, n.BinDir)
+			_ = delPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n.BinDir)
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
@@ -552,7 +615,11 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			logging.Debugf("Marked interface %v for gateway deletion", ifName)
 		} else {
 			// Otherwise, determine if this interface now gets our default route.
-			if delegate.GatewayRequest != nil {
+			// According to 
+			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+			// else we'll update the defaultgateway to the one specified.
+			if delegate.GatewayRequest != nil && delegate.GatewayRequest[0] != nil {
 				deletegateway = true
 				adddefaultgateway = true
 				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
@@ -579,24 +646,39 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			result = tmpResult
 		}
 
-		//create the network status, only in case Multus as kubeconfig
+		// Read devInfo from CNIDeviceInfoFile if it exists so
+		// it can be copied to the NetworkStatus.
+		devinfo, err := getDelegateDeviceInfo(delegate, rt)
+		if err != nil {
+			// Even if the filename is set, file may not be present. Ignore error,
+			// but log and in the future may need to filter on specific errors.
+			logging.Debugf("cmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
+		}
+
+		// create the network status, only in case Multus as kubeconfig
 		if n.Kubeconfig != "" && kc != nil {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
-				delegateNetStatus, err := nadutils.CreateNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin)
+				delegateNetStatus, err := nadutils.CreateNetworkStatus(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
 				if err != nil {
 					return nil, cmdErr(k8sArgs, "error setting network status: %v", err)
 				}
 
 				netStatus = append(netStatus, *delegateNetStatus)
 			}
+		} else if devinfo != nil {
+			// Warn that devinfo exists but could not add it to downwards API
+			logging.Errorf("devinfo available, but no kubeConfig so NetworkStatus not modified.")
 		}
 	}
 
-	//set the network status annotation in apiserver, only in case Multus as kubeconfig
+	// set the network status annotation in apiserver, only in case Multus as kubeconfig
 	if n.Kubeconfig != "" && kc != nil {
 		if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
 			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
+				if strings.Contains(err.Error(), "failed to query the pod") {
+					return nil, cmdErr(k8sArgs, "error setting the networks status, pod was already deleted: %v", err)
+				}
 				return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 			}
 		}
@@ -605,9 +687,10 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	return result, nil
 }
 
-func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+//CmdCheck ...
+func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
-	logging.Debugf("cmdCheck: %v, %v, %v", args, exec, kubeClient)
+	logging.Debugf("CmdCheck: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
 		return err
 	}
@@ -620,8 +703,7 @@ func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 	for idx, delegate := range in.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
 
-		runtimeConfig := types.MergeCNIRuntimeConfig(in.RuntimeConfig, delegate)
-		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, runtimeConfig)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
 		err = delegateCheck(exec, ifName, delegate, rt, in.BinDir)
 		if err != nil {
 			return err
@@ -631,9 +713,10 @@ func cmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 	return nil
 }
 
-func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+//CmdDel ...
+func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
-	logging.Debugf("cmdDel: %v, %v, %v", args, exec, kubeClient)
+	logging.Debugf("CmdDel: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
 		return err
 	}
@@ -647,7 +730,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		_, ok := err.(ns.NSPathNotExistErr)
 		if ok {
 			netnsfound = false
-			logging.Debugf("cmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
+			logging.Debugf("CmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
 		} else {
 			return cmdErr(nil, "failed to open netns %q: %v", netns, err)
 		}
@@ -672,13 +755,45 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
+	kubeClient, err = k8s.GetK8sClient(in.Kubeconfig, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error getting k8s client: %v", err)
+	}
+
+	pod := (*v1.Pod)(nil)
+	if kubeClient != nil {
+		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		if err != nil {
+			var waitErr error
+			// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval
+			if errors.IsServiceUnavailable(err) {
+				pollDuration := 500 * time.Millisecond
+				pollTimeout := 5 * time.Second
+				waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
+					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+					return pod != nil, err
+				})
+				// retry failed, then return error with retry out
+				if waitErr != nil {
+					return cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err)
+				}
+			} else if errors.IsNotFound(err) {
+				// If not found, proceed to remove interface with cache
+				pod = nil
+			} else {
+				// Other case, return error
+				return cmdErr(k8sArgs, "error getting pod: %v", err)
+			}
+		}
+	}
+
 	// Read the cache to get delegates json for the pod
 	netconfBytes, path, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
 	if err != nil {
-		// Fetch delegates again if cache is not exist
-		if os.IsNotExist(err) {
+		// Fetch delegates again if cache is not exist and pod info can be read
+		if os.IsNotExist(err) && pod != nil {
 			if in.ClusterNetwork != "" {
-				err = k8s.GetDefaultNetworks(k8sArgs, in, kubeClient)
+				_, err = k8s.GetDefaultNetworks(pod, in, kubeClient, nil)
 				if err != nil {
 					return cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err)
 				}
@@ -687,7 +802,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 			}
 
 			// Get pod annotation and so on
-			_, _, _, err := k8s.TryLoadPodDelegates(k8sArgs, in, kubeClient)
+			_, _, err := k8s.TryLoadPodDelegates(pod, in, kubeClient, nil)
 			if err != nil {
 				if len(in.Delegates) == 0 {
 					// No delegate available so send error
@@ -697,7 +812,10 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 				logging.Errorf("Multus: failed to get delegates: %v, but continue to delete clusterNetwork", err)
 			}
 		} else {
-			return cmdErr(k8sArgs, "error reading the delegates: %v", err)
+			// The options to continue with a delete have been exhausted (cachefile + API query didn't work)
+			// We cannot exit with an error as this may cause a sandbox to never get deleted.
+			logging.Errorf("Multus: failed to get the cached delegates file: %v, cannot properly delete", err)
+			return nil
 		}
 	} else {
 		defer os.Remove(path)
@@ -737,20 +855,10 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	kubeClient, err = k8s.GetK8sClient(in.Kubeconfig, kubeClient)
-	var pod *v1.Pod
-	if kubeClient != nil {
-		podName := string(k8sArgs.K8S_POD_NAME)
-		podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
-		pod, _ = kubeClient.GetPod(podNamespace, podName)
-	}
-
-	rt := types.CreateCNIRuntimeConf(args, k8sArgs, "", in.RuntimeConfig)
-	return delPlugins(exec, pod, args.IfName, in.Delegates, len(in.Delegates)-1, rt, in.BinDir)
+	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in.BinDir)
 }
 
 func main() {
-
 	// Init command line flags to clear vendored packages' one, especially in init()
 	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
 
@@ -760,21 +868,21 @@ func main() {
 	flag.BoolVar(&versionOpt, "v", false, "Show application version")
 	flag.Parse()
 	if versionOpt == true {
-		fmt.Printf("%s\n", printVersionString())
+		fmt.Printf("%s\n", PrintVersionString())
 		return
 	}
 
 	skel.PluginMain(
 		func(args *skel.CmdArgs) error {
-			result, err := cmdAdd(args, nil, nil)
+			result, err := CmdAdd(args, nil, nil)
 			if err != nil {
 				return err
 			}
 			return result.Print()
 		},
 		func(args *skel.CmdArgs) error {
-			return cmdCheck(args, nil, nil)
+			return CmdCheck(args, nil, nil)
 		},
-		func(args *skel.CmdArgs) error { return cmdDel(args, nil, nil) },
+		func(args *skel.CmdArgs) error { return CmdDel(args, nil, nil) },
 		cniversion.All, "meta-plugin that delegates to other CNI plugins")
 }

pkg/multus/multus_test.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-package main
+package multus
 
 import (
 	"bytes"
@@ -34,11 +34,13 @@ import (
 	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/intel/multus-cni.v3/k8sclient"
-	"gopkg.in/intel/multus-cni.v3/logging"
-	testhelpers "gopkg.in/intel/multus-cni.v3/testing"
-	"gopkg.in/intel/multus-cni.v3/types"
+	"gopkg.in/intel/multus-cni.v3/pkg/k8sclient"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	testhelpers "gopkg.in/intel/multus-cni.v3/pkg/testing"
+	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
 
@@ -293,7 +295,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -302,7 +304,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -369,7 +371,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -378,7 +380,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -389,7 +391,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -446,7 +448,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -454,12 +456,12 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 
 		// Check is not supported until v 0.4.0
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -526,7 +528,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -535,7 +537,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -546,7 +548,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -603,7 +605,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -611,12 +613,12 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 
 		// Check is not supported until v 0.4.0
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -684,7 +686,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -693,7 +695,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -704,7 +706,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -761,7 +763,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -769,12 +771,12 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 
 		// Check is not supported until v 0.4.0
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -841,7 +843,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -850,7 +852,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -861,7 +863,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -918,7 +920,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -926,12 +928,12 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 
 		// Check is not supported until v 0.4.0
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -998,7 +1000,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1007,7 +1009,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -1018,7 +1020,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -1073,7 +1075,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1081,12 +1083,12 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 
 		// Check is not supported until v 0.4.0
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -1097,7 +1099,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}
 	})
 
-	It("fails to load NetConf with bad json in cmdAdd/Del", func() {
+	It("fails to load NetConf with bad json in CmdAdd/Del", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -1153,10 +1155,10 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err := cmdAdd(args, fExec, nil)
+		_, err := CmdAdd(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -1223,7 +1225,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err = cmdAdd(args, fExec, nil)
+		_, err = CmdAdd(args, fExec, nil)
 		Expect(fExec.addIndex).To(Equal(2))
 		Expect(fExec.delIndex).To(Equal(2))
 		Expect(err).To(MatchError("[/:other1]: error adding container to network \"other1\": expected plugin failure"))
@@ -1247,7 +1249,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		    "cniVersion": "0.2.0",
 		    "type": "other-plugin"
 		}`
-		// took out the name in expectedConf2, expecting a new value to be filled in by cmdAdd
+		// took out the name in expectedConf2, expecting a new value to be filled in by CmdAdd
 
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -1281,7 +1283,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err = cmdAdd(args, fExec, nil)
+		_, err = CmdAdd(args, fExec, nil)
 		Expect(fExec.addIndex).To(Equal(2))
 		Expect(fExec.delIndex).To(Equal(2))
 		Expect(err).To(HaveOccurred())
@@ -1392,7 +1394,8 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -1401,7 +1404,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*current.Result)
@@ -1470,7 +1473,8 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -1486,7 +1490,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1497,8 +1501,8 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		events := collectEvents(recorder.Events)
 		Expect(len(events)).To(Equal(3))
 		Expect(events[0]).To(Equal("Normal AddedInterface Add eth0 [1.1.1.2/24]"))
-		Expect(events[1]).To(Equal("Normal AddedInterface Add net1 [1.1.1.3/24] from net1"))
-		Expect(events[2]).To(Equal("Normal AddedInterface Add net2 [1.1.1.4/24] from net2"))
+		Expect(events[1]).To(Equal("Normal AddedInterface Add net1 [1.1.1.3/24] from test/net1"))
+		Expect(events[2]).To(Equal("Normal AddedInterface Add net2 [1.1.1.4/24] from test/net2"))
 	})
 
 	It("executes kubernetes networks and delete it after pod removal", func() {
@@ -1546,7 +1550,8 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -1555,7 +1560,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1564,9 +1569,15 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		// set fKubeClient to nil to emulate no pod info
+
+		// delete pod to emulate no pod info
 		clientInfo.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, clientInfo)
+		nilPod, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Get(
+			context.TODO(), fakePod.ObjectMeta.Name, metav1.GetOptions{})
+		Expect(nilPod).To(BeNil())
+		Expect(errors.IsNotFound(err)).To(BeTrue())
+
+		err = CmdDel(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -1623,7 +1634,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1634,7 +1645,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		os.Setenv("CNI_IFNAME", "eth0")
 		// set fKubeClient to nil to emulate no pod info
 		fKubeClient.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -1691,7 +1702,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1702,7 +1713,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		os.Setenv("CNI_IFNAME", "eth0")
 		// set fKubeClient to nil to emulate no pod info
 		fKubeClient.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -1748,7 +1759,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		fExec.addPlugin020(nil, "eth0", expectedConf1, nil, nil)
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err := cmdAdd(args, fExec, nil)
+		_, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -1790,7 +1801,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1798,7 +1809,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -1859,7 +1870,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(err).NotTo(HaveOccurred())
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1874,7 +1885,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		By("Delete and check net count is not incremented")
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -1935,7 +1946,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(err).NotTo(HaveOccurred())
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*types020.Result)
@@ -1953,7 +1964,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		By("Delete and check pod/net count is incremented")
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -2019,7 +2030,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
 		k8sargs, err := k8sclient.GetK8sArgs(args)
 		n, err := types.LoadNetConf(args.StdinData)
-		rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
 
 		err = conflistDel(rt, rawnetconflist, binDir, fExec)
 		Expect(err).To(HaveOccurred())
@@ -2070,8 +2081,9 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		fExec.addPlugin020(nil, "eth0", expectedConf1, nil, nil)
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-
-		err := cmdDel(args, fExec, nil)
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })
@@ -2160,7 +2172,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
@@ -2168,12 +2180,12 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
 
 		os.Setenv("CNI_COMMAND", "CHECK")
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -2242,7 +2254,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -2250,7 +2262,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -2261,7 +2273,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		}
 	})
 
-	It("returns the previous result using cmdCheck", func() {
+	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -2328,7 +2340,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, nil)
+		result, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
@@ -2336,12 +2348,12 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
 
 		os.Setenv("CNI_COMMAND", "CHECK")
-		err = cmdCheck(args, fExec, nil)
+		err = CmdCheck(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 
@@ -2352,7 +2364,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		}
 	})
 
-	It("fails to load NetConf with bad json in cmdAdd/Del", func() {
+	It("fails to load NetConf with bad json in CmdAdd/Del", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -2410,10 +2422,10 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err := cmdAdd(args, fExec, nil)
+		_, err := CmdAdd(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 
-		err = cmdDel(args, fExec, nil)
+		err = CmdDel(args, fExec, nil)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -2461,7 +2473,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err = cmdAdd(args, fExec, nil)
+		_, err = CmdAdd(args, fExec, nil)
 		Expect(fExec.addIndex).To(Equal(2))
 		Expect(fExec.delIndex).To(Equal(2))
 		Expect(err).To(MatchError("[/:other1]: error adding container to network \"other1\": expected plugin failure"))
@@ -2485,7 +2497,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		    "cniVersion": "0.4.0",
 		    "type": "other-plugin"
 		}`
-		// took out the name in expectedConf2, expecting a new value to be filled in by cmdAdd
+		// took out the name in expectedConf2, expecting a new value to be filled in by CmdAdd
 
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -2520,7 +2532,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err = cmdAdd(args, fExec, nil)
+		_, err = CmdAdd(args, fExec, nil)
 		Expect(fExec.addIndex).To(Equal(2))
 		Expect(fExec.delIndex).To(Equal(2))
 		Expect(err).To(HaveOccurred())
@@ -2631,7 +2643,8 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -2640,7 +2653,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		r := result.(*current.Result)
@@ -2713,7 +2726,8 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -2729,7 +2743,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -2783,7 +2797,8 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		}, nil)
 
 		clientInfo := NewFakeClientInfo()
-		_, err := clientInfo.Client.Core().Pods(fakePod.ObjectMeta.Namespace).Create(fakePod)
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = clientInfo.AddNetAttachDef(
@@ -2792,7 +2807,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, clientInfo)
+		result, err := CmdAdd(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -2802,7 +2817,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		os.Setenv("CNI_IFNAME", "eth0")
 		// set fKubeClient to nil to emulate no pod info
 		clientInfo.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, clientInfo)
+		err = CmdDel(args, fExec, clientInfo)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -2861,7 +2876,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -2871,7 +2886,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		os.Setenv("CNI_IFNAME", "eth0")
 		// set fKubeClient to nil to emulate no pod info
 		fKubeClient.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -2930,7 +2945,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -2940,7 +2955,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		os.Setenv("CNI_IFNAME", "eth0")
 		// set fKubeClient to nil to emulate no pod info
 		fKubeClient.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -2986,7 +3001,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		fExec.addPlugin(nil, "eth0", expectedConf1, nil, nil)
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		_, err := cmdAdd(args, fExec, nil)
+		_, err := CmdAdd(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -3029,14 +3044,14 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
 
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -3099,7 +3114,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(err).NotTo(HaveOccurred())
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -3113,7 +3128,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		By("Delete and check net count is not incremented")
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -3176,7 +3191,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(err).NotTo(HaveOccurred())
 		os.Setenv("CNI_COMMAND", "ADD")
 		os.Setenv("CNI_IFNAME", "eth0")
-		result, err := cmdAdd(args, fExec, fKubeClient)
+		result, err := CmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
 		// plugin 1 is the masterplugin
@@ -3193,7 +3208,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		By("Delete and check pod/net count is incremented")
 		os.Setenv("CNI_COMMAND", "DEL")
 		os.Setenv("CNI_IFNAME", "eth0")
-		err = cmdDel(args, fExec, fKubeClient)
+		err = CmdDel(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
@@ -3261,7 +3276,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`)
 		k8sargs, err := k8sclient.GetK8sArgs(args)
 		n, err := types.LoadNetConf(args.StdinData)
-		rt := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
 
 		err = conflistDel(rt, rawnetconflist, binDir, fExec)
 		Expect(err).To(HaveOccurred())

pkg/netutils/netutils.go

@@ -20,7 +20,7 @@ import (
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ns"
-	"gopkg.in/intel/multus-cni.v3/logging"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
 	"github.com/vishvananda/netlink"
 	"net"
 	"strings"

pkg/types/conf.go

@@ -17,13 +17,16 @@ package types
 
 import (
 	"encoding/json"
+	"fmt"
 	"net"
+	"strings"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/cni/pkg/version"
-	"gopkg.in/intel/multus-cni.v3/logging"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
+	"gopkg.in/intel/multus-cni.v3/pkg/logging"
 )
 
 const (
@@ -32,6 +35,7 @@ const (
 	defaultBinDir                 = "/opt/cni/bin"
 	defaultReadinessIndicatorFile = ""
 	defaultMultusNamespace        = "kube-system"
+	defaultNonIsolatedNamespace   = "default"
 )
 
 // LoadDelegateNetConfList reads DelegateNetConf from bytes
@@ -54,7 +58,7 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 }
 
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
-func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string) (*DelegateNetConf, error) {
+func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
 	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
 
@@ -86,6 +90,9 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConf bytes: %v", err)
 			}
+			// Save them for housekeeping
+			delegateConf.ResourceName = resourceName
+			delegateConf.DeviceID = deviceID
 		}
 		if net != nil && net.CNIArgs != nil {
 			bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs)
@@ -97,7 +104,8 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 
 	if net != nil {
 		if net.Name != "" {
-			delegateConf.Name = net.Name
+			// Overwrite CNI config name with net-attach-def name
+			delegateConf.Name = fmt.Sprintf("%s/%s", net.Namespace, net.Name)
 		}
 		if net.InterfaceRequest != "" {
 			delegateConf.IfnameRequest = net.InterfaceRequest
@@ -117,6 +125,16 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 		if net.GatewayRequest != nil {
 			delegateConf.GatewayRequest = append(delegateConf.GatewayRequest, net.GatewayRequest...)
 		}
+		if net.InfinibandGUIDRequest != "" {
+			delegateConf.InfinibandGUIDRequest = net.InfinibandGUIDRequest
+		}
+		if net.DeviceID != "" {
+			if deviceID != "" {
+				logging.Debugf("Warning: Both RuntimeConfig and ResourceMap provide deviceID. Ignoring RuntimeConfig")
+			} else {
+				delegateConf.DeviceID = net.DeviceID
+			}
+		}
 	}
 
 	delegateConf.Bytes = bytes
@@ -124,36 +142,62 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 	return delegateConf, nil
 }
 
-// MergeCNIRuntimeConfig creates CNI runtimeconfig from delegate
-func MergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetConf) *RuntimeConfig {
-	logging.Debugf("MergeCNIRuntimeConfig: %v %v", runtimeConfig, delegate)
+// mergeCNIRuntimeConfig creates CNI runtimeconfig from delegate
+func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetConf) *RuntimeConfig {
+	logging.Debugf("mergeCNIRuntimeConfig: %v %v", runtimeConfig, delegate)
+	var mergedRuntimeConfig RuntimeConfig
+
 	if runtimeConfig == nil {
-		runtimeConfig = &RuntimeConfig{}
+		mergedRuntimeConfig = RuntimeConfig{}
+	} else {
+		mergedRuntimeConfig = *runtimeConfig
 	}
 
 	// multus inject RuntimeConfig only in case of non MasterPlugin.
 	if delegate.MasterPlugin != true {
-		logging.Debugf("MergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", runtimeConfig)
+		logging.Debugf("mergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", mergedRuntimeConfig)
 		if delegate.PortMappingsRequest != nil {
-			runtimeConfig.PortMaps = delegate.PortMappingsRequest
+			mergedRuntimeConfig.PortMaps = delegate.PortMappingsRequest
 		}
 		if delegate.BandwidthRequest != nil {
-			runtimeConfig.Bandwidth = delegate.BandwidthRequest
+			mergedRuntimeConfig.Bandwidth = delegate.BandwidthRequest
 		}
 		if delegate.IPRequest != nil {
-			runtimeConfig.IPs = delegate.IPRequest
+			mergedRuntimeConfig.IPs = delegate.IPRequest
 		}
 		if delegate.MacRequest != "" {
-			runtimeConfig.Mac = delegate.MacRequest
+			mergedRuntimeConfig.Mac = delegate.MacRequest
 		}
+		if delegate.InfinibandGUIDRequest != "" {
+			mergedRuntimeConfig.InfinibandGUID = delegate.InfinibandGUIDRequest
+		}
+		if delegate.DeviceID != "" {
+			mergedRuntimeConfig.DeviceID = delegate.DeviceID
+		}
+		logging.Debugf("mergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", mergedRuntimeConfig)
 	}
-
-	return runtimeConfig
+	return &mergedRuntimeConfig
 }
 
-// CreateCNIRuntimeConf create CNI RuntimeConf
-func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig) *libcni.RuntimeConf {
-	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v", args, k8sArgs, ifName, rc)
+// CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
+// exists, merge data with the runtime config.
+func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
+	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
+	var cniDeviceInfoFile string
+	var delegateRc *RuntimeConfig
+
+	if delegate != nil {
+		delegateRc = mergeCNIRuntimeConfig(rc, delegate)
+		if delegateRc.CNIDeviceInfoFile != "" {
+			logging.Debugf("Warning: Existing value of CNIDeviceInfoFile will be overwritten %s", delegateRc.CNIDeviceInfoFile)
+		}
+		autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName)
+		delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo)
+		cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile
+		logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile)
+	} else {
+		delegateRc = rc
+	}
 
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
 	// Todo
@@ -171,23 +215,32 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 		},
 	}
 
-	if rc != nil {
+	if delegateRc != nil {
 		capabilityArgs := map[string]interface{}{}
-		if len(rc.PortMaps) != 0 {
-			capabilityArgs["portMappings"] = rc.PortMaps
+		if len(delegateRc.PortMaps) != 0 {
+			capabilityArgs["portMappings"] = delegateRc.PortMaps
 		}
-		if rc.Bandwidth != nil {
-			capabilityArgs["bandwidth"] = rc.Bandwidth
+		if delegateRc.Bandwidth != nil {
+			capabilityArgs["bandwidth"] = delegateRc.Bandwidth
 		}
-		if len(rc.IPs) != 0 {
-			capabilityArgs["ips"] = rc.IPs
+		if len(delegateRc.IPs) != 0 {
+			capabilityArgs["ips"] = delegateRc.IPs
 		}
-		if len(rc.Mac) != 0 {
-			capabilityArgs["mac"] = rc.Mac
+		if len(delegateRc.Mac) != 0 {
+			capabilityArgs["mac"] = delegateRc.Mac
+		}
+		if len(delegateRc.InfinibandGUID) != 0 {
+			capabilityArgs["infinibandGUID"] = delegateRc.InfinibandGUID
+		}
+		if delegateRc.DeviceID != "" {
+			capabilityArgs["deviceID"] = delegateRc.DeviceID
+		}
+		if delegateRc.CNIDeviceInfoFile != "" {
+			capabilityArgs["CNIDeviceInfoFile"] = delegateRc.CNIDeviceInfoFile
 		}
 		rt.CapabilityArgs = capabilityArgs
 	}
-	return rt
+	return rt, cniDeviceInfoFile
 }
 
 // GetGatewayFromResult retrieves gateway IP addresses from CNI result
@@ -204,7 +257,8 @@ func GetGatewayFromResult(result *current.Result) []net.IP {
 
 // LoadNetConf converts inputs (i.e. stdin) to NetConf
 func LoadNetConf(bytes []byte) (*NetConf, error) {
-	netconf := &NetConf{}
+	// LogToStderr's default value set to true
+	netconf := &NetConf{LogToStderr: true}
 
 	logging.Debugf("LoadNetConf: %s", string(bytes))
 	if err := json.Unmarshal(bytes, netconf); err != nil {
@@ -212,6 +266,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	}
 
 	// Logging
+	logging.SetLogStderr(netconf.LogToStderr)
 	if netconf.LogFile != "" {
 		logging.SetLogFile(netconf.LogFile)
 	}
@@ -270,6 +325,19 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 		netconf.MultusNamespace = defaultMultusNamespace
 	}
 
+	// setup namespace isolation
+	if netconf.RawNonIsolatedNamespaces == "" {
+		netconf.NonIsolatedNamespaces = []string{defaultNonIsolatedNamespace}
+	} else {
+		// Parse the comma separated list
+		nonisolated := strings.Split(netconf.RawNonIsolatedNamespaces, ",")
+		// Cleanup the whitespace
+		for i, nonv := range nonisolated {
+			nonisolated[i] = strings.TrimSpace(nonv)
+		}
+		netconf.NonIsolatedNamespaces = nonisolated
+	}
+
 	// get RawDelegates and put delegates field
 	if netconf.ClusterNetwork == "" {
 		// for Delegates
@@ -281,7 +349,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 			if err != nil {
 				return nil, logging.Errorf("LoadNetConf: error marshalling delegate %d config: %v", idx, err)
 			}
-			delegateConf, err := LoadDelegateNetConf(bytes, nil, "")
+			delegateConf, err := LoadDelegateNetConf(bytes, nil, "", "")
 			if err != nil {
 				return nil, logging.Errorf("LoadNetConf: failed to load delegate %d config: %v", idx, err)
 			}

pkg/types/conf_test.go

@@ -26,8 +26,8 @@ import (
 	types020 "github.com/containernetworking/cni/pkg/types/020"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	testhelpers "gopkg.in/intel/multus-cni.v3/testing"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
+	testhelpers "gopkg.in/intel/multus-cni.v3/pkg/testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -102,7 +102,7 @@ var _ = Describe("config operations", func() {
 
 		_, err := LoadNetConf([]byte(conf))
 		Expect(err).To(HaveOccurred())
-		_, err = LoadDelegateNetConf([]byte(conf), nil, "")
+		_, err = LoadDelegateNetConf([]byte(conf), nil, "", "")
 		Expect(err).To(HaveOccurred())
 		err = LoadDelegateNetConfList([]byte(conf), &DelegateNetConf{})
 		Expect(err).To(HaveOccurred())
@@ -135,6 +135,47 @@ var _ = Describe("config operations", func() {
 		Expect(netConf.LogFile).To(Equal("/var/log/multus.log"))
 	})
 
+	It("properly sets namespace isolation using the default namespace", func() {
+		conf := `{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "logLevel": "debug",
+	    "logFile": "/var/log/multus.log",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "namespaceIsolation": true,
+	    "delegates": [{
+	        "type": "weave-net"
+	    }]
+	}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netConf.NamespaceIsolation).To(Equal(true))
+		Expect(len(netConf.NonIsolatedNamespaces)).To(Equal(1))
+		Expect(netConf.NonIsolatedNamespaces[0]).To(Equal("default"))
+	})
+
+	It("properly sets namespace isolation using custom namespaces", func() {
+		conf := `{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "logLevel": "debug",
+	    "logFile": "/var/log/multus.log",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "namespaceIsolation": true,
+	    "globalNamespaces": " foo,bar ,default",
+	    "delegates": [{
+	        "type": "weave-net"
+	    }]
+	}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netConf.NamespaceIsolation).To(Equal(true))
+		Expect(len(netConf.NonIsolatedNamespaces)).To(Equal(3))
+		Expect(netConf.NonIsolatedNamespaces[0]).To(Equal("foo"))
+		Expect(netConf.NonIsolatedNamespaces[1]).To(Equal("bar"))
+		Expect(netConf.NonIsolatedNamespaces[2]).To(Equal("default"))
+	})
+
 	It("prevResult with no errors", func() {
 		conf := `{
 	    "name": "node-cni-network",
@@ -280,7 +321,7 @@ var _ = Describe("config operations", func() {
 			DeviceID string `json:"deviceID"`
 		}
 		sriovConf := &sriovNetConf{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConf)
@@ -306,7 +347,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*sriovNetConf `json:"plugins"`
 		}
 		sriovConfList := &sriovNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -335,7 +376,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*sriovNetConf `json:"plugins"`
 		}
 		sriovConfList := &sriovNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.1", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &sriovConfList)
@@ -356,7 +397,7 @@ var _ = Describe("config operations", func() {
 			PCIBusID string `json:"pciBusID"`
 		}
 		hostDeviceConf := &hostDeviceNetConf{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.2", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConf)
@@ -382,7 +423,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*hostDeviceNetConf `json:"plugins"`
 		}
 		hostDeviceConfList := &hostDeviceNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -411,7 +452,7 @@ var _ = Describe("config operations", func() {
 			Plugins []*hostDeviceNetConf `json:"plugins"`
 		}
 		hostDeviceConfList := &hostDeviceNetConfList{}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.3", "")
 		Expect(err).NotTo(HaveOccurred())
 
 		err = json.Unmarshal(delegateNetConf.Bytes, &hostDeviceConfList)
@@ -444,7 +485,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConf := &bridgeNetConf{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -480,7 +521,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConf := &bridgeNetConf{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConf)
@@ -518,7 +559,7 @@ var _ = Describe("config operations", func() {
 			Name:    "test-elem",
 			CNIArgs: &args,
 		}
-		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "")
+		delegateNetConf, err := LoadDelegateNetConf([]byte(conf), net, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		bridgeConflist := &bridgeNetConfList{}
 		err = json.Unmarshal(delegateNetConf.Bytes, bridgeConflist)
@@ -565,7 +606,7 @@ var _ = Describe("config operations", func() {
 			HostIP:        "anotherSampleHostIP",
 		}
 
-		rt := CreateCNIRuntimeConf(args, k8sArgs, "", rc)
+		rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", rc, nil)
 		fmt.Println("rt.ContainerID: ", rt.ContainerID)
 		Expect(rt.ContainerID).To(Equal("123456789"))
 		Expect(rt.NetNS).To(Equal(args.Netns))
@@ -595,10 +636,10 @@ var _ = Describe("config operations", func() {
     }
 }`
 
-		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
 		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 
@@ -628,10 +669,10 @@ var _ = Describe("config operations", func() {
     }
 }`
 
-		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0")
+		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 		fmt.Println("result.Version: ", result.Version())
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin)
+		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
 		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 
@@ -659,24 +700,26 @@ var _ = Describe("config operations", func() {
 		}
 
 		networkSelection := &NetworkSelectionElement{
-			Name:                "testname",
-			InterfaceRequest:    "testIF1",
-			MacRequest:          "c2:11:22:33:44:66",
-			IPRequest:           []string{"10.0.0.1/24"},
-			BandwidthRequest:    bandwidthEntry1,
-			PortMappingsRequest: []*PortMapEntry{portMapEntry1},
+			Name:                  "testname",
+			InterfaceRequest:      "testIF1",
+			MacRequest:            "c2:11:22:33:44:66",
+			InfinibandGUIDRequest: "24:8a:07:03:00:8d:ae:2e",
+			IPRequest:             []string{"10.0.0.1/24"},
+			BandwidthRequest:      bandwidthEntry1,
+			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
 
-		delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "")
+		delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "", "")
 		Expect(err).NotTo(HaveOccurred())
 		Expect(delegateConf.IfnameRequest).To(Equal(networkSelection.InterfaceRequest))
 		Expect(delegateConf.MacRequest).To(Equal(networkSelection.MacRequest))
+		Expect(delegateConf.InfinibandGUIDRequest).To(Equal(networkSelection.InfinibandGUIDRequest))
 		Expect(delegateConf.IPRequest).To(Equal(networkSelection.IPRequest))
 		Expect(delegateConf.BandwidthRequest).To(Equal(networkSelection.BandwidthRequest))
 		Expect(delegateConf.PortMappingsRequest).To(Equal(networkSelection.PortMappingsRequest))
 	})
 
-	It("test MergeCNIRuntimeConfig with masterPlugin", func() {
+	It("test mergeCNIRuntimeConfig with masterPlugin", func() {
 		conf := `{
 			"name": "node-cni-network",
 			"type": "multus",
@@ -704,22 +747,27 @@ var _ = Describe("config operations", func() {
 		}
 
 		networkSelection := &NetworkSelectionElement{
-			Name:                "testname",
-			InterfaceRequest:    "testIF1",
-			MacRequest:          "c2:11:22:33:44:66",
-			IPRequest:           []string{"10.0.0.1/24"},
-			BandwidthRequest:    bandwidthEntry1,
-			PortMappingsRequest: []*PortMapEntry{portMapEntry1},
+			Name:                  "testname",
+			InterfaceRequest:      "testIF1",
+			MacRequest:            "c2:11:22:33:44:66",
+			InfinibandGUIDRequest: "24:8a:07:03:00:8d:ae:2e",
+			IPRequest:             []string{"10.0.0.1/24"},
+			BandwidthRequest:      bandwidthEntry1,
+			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
-		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
+		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
 		delegate.MasterPlugin = true
+		origRuntimeConfig := RuntimeConfig{}
 		Expect(err).NotTo(HaveOccurred())
-		runtimeConf := MergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
+		runtimeConf := mergeCNIRuntimeConfig(&origRuntimeConfig, delegate)
 		Expect(runtimeConf.PortMaps).To(BeNil())
 		Expect(runtimeConf.Bandwidth).To(BeNil())
+		Expect(runtimeConf.InfinibandGUID).To(Equal(""))
+		// The original RuntimeConfig must have not been overwritten
+		Expect(origRuntimeConfig).To(Equal(RuntimeConfig{}))
 	})
 
-	It("test MergeCNIRuntimeConfig with delegate plugin", func() {
+	It("test mergeCNIRuntimeConfig with delegate plugin", func() {
 		conf := `{
 			"name": "weave1",
 			"cniVersion": "0.2.0",
@@ -739,21 +787,26 @@ var _ = Describe("config operations", func() {
 		}
 
 		networkSelection := &NetworkSelectionElement{
-			Name:                "testname",
-			InterfaceRequest:    "testIF1",
-			MacRequest:          "c2:11:22:33:44:66",
-			IPRequest:           []string{"10.0.0.1/24"},
-			BandwidthRequest:    bandwidthEntry1,
-			PortMappingsRequest: []*PortMapEntry{portMapEntry1},
+			Name:                  "testname",
+			InterfaceRequest:      "testIF1",
+			MacRequest:            "c2:11:22:33:44:66",
+			InfinibandGUIDRequest: "24:8a:07:03:00:8d:ae:2e",
+			IPRequest:             []string{"10.0.0.1/24"},
+			BandwidthRequest:      bandwidthEntry1,
+			PortMappingsRequest:   []*PortMapEntry{portMapEntry1},
 		}
-		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "")
+		origRuntimeConfig := RuntimeConfig{}
+		delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "", "")
 		Expect(err).NotTo(HaveOccurred())
-		runtimeConf := MergeCNIRuntimeConfig(&RuntimeConfig{}, delegate)
+		runtimeConf := mergeCNIRuntimeConfig(&origRuntimeConfig, delegate)
 		Expect(runtimeConf.PortMaps).NotTo(BeNil())
 		Expect(len(runtimeConf.PortMaps)).To(BeEquivalentTo(1))
 		Expect(runtimeConf.PortMaps[0]).To(Equal(portMapEntry1))
 		Expect(runtimeConf.Bandwidth).To(Equal(bandwidthEntry1))
 		Expect(len(runtimeConf.IPs)).To(BeEquivalentTo(1))
 		Expect(runtimeConf.Mac).To(Equal("c2:11:22:33:44:66"))
+		Expect(runtimeConf.InfinibandGUID).To(Equal("24:8a:07:03:00:8d:ae:2e"))
+		// The original RuntimeConfig must have not been overwritten
+		Expect(origRuntimeConfig).To(Equal(RuntimeConfig{}))
 	})
 })

pkg/types/types.go

@@ -38,17 +38,20 @@ type NetConf struct {
 	// RawDelegates is private to the NetConf class; use Delegates instead
 	RawDelegates    []map[string]interface{} `json:"delegates"`
 	Delegates       []*DelegateNetConf       `json:"-"`
-	NetStatus       []*NetworkStatus         `json:"-"`
 	Kubeconfig      string                   `json:"kubeconfig"`
 	ClusterNetwork  string                   `json:"clusterNetwork"`
 	DefaultNetworks []string                 `json:"defaultNetworks"`
 	LogFile         string                   `json:"logFile"`
 	LogLevel        string                   `json:"logLevel"`
+	LogToStderr     bool                     `json:"logToStderr,omitempty"`
 	RuntimeConfig   *RuntimeConfig           `json:"runtimeConfig,omitempty"`
 	// Default network readiness options
 	ReadinessIndicatorFile string `json:"readinessindicatorfile"`
 	// Option to isolate the usage of CR's to the namespace in which a pod resides.
-	NamespaceIsolation bool `json:"namespaceIsolation"`
+	NamespaceIsolation       bool     `json:"namespaceIsolation"`
+	RawNonIsolatedNamespaces string   `json:"globalNamespaces"`
+	NonIsolatedNamespaces    []string `json:"-"`
+
 	// Option to set system namespaces (to avoid to add defaultNetworks)
 	SystemNamespaces []string `json:"systemNamespaces"`
 	// Option to set the namespace that multus-cni uses (clusterNetwork/defaultNetworks)
@@ -57,10 +60,13 @@ type NetConf struct {
 
 // RuntimeConfig specifies CNI RuntimeConfig
 type RuntimeConfig struct {
-	PortMaps  []*PortMapEntry `json:"portMappings,omitempty"`
-	Bandwidth *BandwidthEntry `json:"bandwidth,omitempty"`
-	IPs       []string        `json:"ips,omitempty"`
-	Mac       string          `json:"mac,omitempty"`
+	PortMaps          []*PortMapEntry `json:"portMappings,omitempty"`
+	Bandwidth         *BandwidthEntry `json:"bandwidth,omitempty"`
+	IPs               []string        `json:"ips,omitempty"`
+	Mac               string          `json:"mac,omitempty"`
+	InfinibandGUID    string          `json:"infinibandGUID,omitempty"`
+	DeviceID          string          `json:"deviceID,omitempty"`
+	CNIDeviceInfoFile string          `json:"CNIDeviceInfoFile,omitempty"`
 }
 
 // PortMapEntry for CNI PortMapEntry
@@ -80,32 +86,27 @@ type BandwidthEntry struct {
 	EgressBurst int `json:"egressBurst"`
 }
 
-// NetworkStatus is for network status annotation for pod
-type NetworkStatus struct {
-	Name      string    `json:"name"`
-	Interface string    `json:"interface,omitempty"`
-	IPs       []string  `json:"ips,omitempty"`
-	Mac       string    `json:"mac,omitempty"`
-	DNS       types.DNS `json:"dns,omitempty"`
-	Gateway   []net.IP  `json:"default-route,omitempty"`
-}
-
 // DelegateNetConf for net-attach-def for pod
 type DelegateNetConf struct {
-	Conf                types.NetConf
-	ConfList            types.NetConfList
-	Name                string
-	IfnameRequest       string          `json:"ifnameRequest,omitempty"`
-	MacRequest          string          `json:"macRequest,omitempty"`
-	IPRequest           []string        `json:"ipRequest,omitempty"`
-	PortMappingsRequest []*PortMapEntry `json:"-"`
-	BandwidthRequest    *BandwidthEntry `json:"-"`
-	GatewayRequest      []net.IP        `json:"default-route,omitempty"`
-	IsFilterGateway     bool
+	Conf                  types.NetConf
+	ConfList              types.NetConfList
+	Name                  string
+	IfnameRequest         string          `json:"ifnameRequest,omitempty"`
+	MacRequest            string          `json:"macRequest,omitempty"`
+	InfinibandGUIDRequest string          `json:"infinibandGUIDRequest,omitempty"`
+	IPRequest             []string        `json:"ipRequest,omitempty"`
+	PortMappingsRequest   []*PortMapEntry `json:"-"`
+	BandwidthRequest      *BandwidthEntry `json:"-"`
+	GatewayRequest        []net.IP        `json:"default-route,omitempty"`
+	IsFilterGateway       bool
 	// MasterPlugin is only used internal housekeeping
 	MasterPlugin bool `json:"-"`
 	// Conflist plugin is only used internal housekeeping
 	ConfListPlugin bool `json:"-"`
+	// DeviceID is only used internal housekeeping
+	DeviceID string `json:"deviceID,omitempty"`
+	// ResourceName is only used internal housekeeping
+	ResourceName string `json:"resourceName,omitempty"`
 
 	// Raw JSON
 	Bytes []byte
@@ -126,6 +127,9 @@ type NetworkSelectionElement struct {
 	// MacRequest contains an optional requested MAC address for this
 	// network attachment
 	MacRequest string `json:"mac,omitempty"`
+	// InfinibandGUID request contains an optional requested Infiniband GUID address
+	// for this network attachment
+	InfinibandGUIDRequest string `json:"infiniband-guid,omitempty"`
 	// InterfaceRequest contains an optional requested name for the
 	// network interface this attachment will create in the container
 	InterfaceRequest string `json:"interface,omitempty"`
@@ -138,6 +142,8 @@ type NetworkSelectionElement struct {
 	// BandwidthRequest contains an optional requested bandwidth for
 	// the network
 	BandwidthRequest *BandwidthEntry `json:"bandwidth,omitempty"`
+	// DeviceID contains an optional requested deviceID the network
+	DeviceID string `json:"deviceID,omitempty"`
 	// CNIArgs contains additional CNI arguments for the network interface
 	CNIArgs *map[string]interface{} `json:"cni-args"`
 	// GatewayRequest contains default route IP address for the pod
@@ -148,9 +154,9 @@ type NetworkSelectionElement struct {
 type K8sArgs struct {
 	types.CommonArgs
 	IP                         net.IP
-	K8S_POD_NAME               types.UnmarshallableString
-	K8S_POD_NAMESPACE          types.UnmarshallableString
-	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString
+	K8S_POD_NAME               types.UnmarshallableString //revive:disable-line
+	K8S_POD_NAMESPACE          types.UnmarshallableString //revive:disable-line
+	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString //revive:disable-line
 }
 
 // ResourceInfo is struct to hold Pod device allocation information

vendor/github.com/evanphx/json-patch/README.md

@@ -28,10 +28,15 @@ go get -u github.com/evanphx/json-patch
 
 # Configuration
 
-There is a single global configuration variable `jsonpatch.SupportNegativeIndices'. This
-defaults to `true` and enables the non-standard practice of allowing negative indices
-to mean indices starting at the end of an array. This functionality can be disabled
-by setting `jsonpatch.SupportNegativeIndices = false`.
+* There is a global configuration variable `jsonpatch.SupportNegativeIndices`.
+  This defaults to `true` and enables the non-standard practice of allowing
+  negative indices to mean indices starting at the end of an array. This
+  functionality can be disabled by setting `jsonpatch.SupportNegativeIndices =
+  false`.
+
+* There is a global configuration variable `jsonpatch.AccumulatedCopySizeLimit`,
+  which limits the total size increase in bytes caused by "copy" operations in a
+  patch. It defaults to 0, which means there is no limit.
 
 ## Create and apply a merge patch
 Given both an original JSON document and a modified JSON document, you can create

vendor/github.com/evanphx/json-patch/errors.go

@@ -0,0 +1,38 @@
+package jsonpatch
+
+import "fmt"
+
+// AccumulatedCopySizeError is an error type returned when the accumulated size
+// increase caused by copy operations in a patch operation has exceeded the
+// limit.
+type AccumulatedCopySizeError struct {
+	limit       int64
+	accumulated int64
+}
+
+// NewAccumulatedCopySizeError returns an AccumulatedCopySizeError.
+func NewAccumulatedCopySizeError(l, a int64) *AccumulatedCopySizeError {
+	return &AccumulatedCopySizeError{limit: l, accumulated: a}
+}
+
+// Error implements the error interface.
+func (a *AccumulatedCopySizeError) Error() string {
+	return fmt.Sprintf("Unable to complete the copy, the accumulated size increase of copy is %d, exceeding the limit %d", a.accumulated, a.limit)
+}
+
+// ArraySizeError is an error type returned when the array size has exceeded
+// the limit.
+type ArraySizeError struct {
+	limit int
+	size  int
+}
+
+// NewArraySizeError returns an ArraySizeError.
+func NewArraySizeError(l, s int) *ArraySizeError {
+	return &ArraySizeError{limit: l, size: s}
+}
+
+// Error implements the error interface.
+func (a *ArraySizeError) Error() string {
+	return fmt.Sprintf("Unable to create array of size %d, limit is %d", a.size, a.limit)
+}

vendor/github.com/evanphx/json-patch/patch.go

@@ -14,7 +14,15 @@ const (
 	eAry
 )
 
-var SupportNegativeIndices bool = true
+var (
+	// SupportNegativeIndices decides whether to support non-standard practice of
+	// allowing negative indices to mean indices starting at the end of an array.
+	// Default to true.
+	SupportNegativeIndices bool = true
+	// AccumulatedCopySizeLimit limits the total size increase in bytes caused by
+	// "copy" operations in a patch.
+	AccumulatedCopySizeLimit int64 = 0
+)
 
 type lazyNode struct {
 	raw   *json.RawMessage
@@ -63,6 +71,20 @@ func (n *lazyNode) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+func deepCopy(src *lazyNode) (*lazyNode, int, error) {
+	if src == nil {
+		return nil, 0, nil
+	}
+	a, err := src.MarshalJSON()
+	if err != nil {
+		return nil, 0, err
+	}
+	sz := len(a)
+	ra := make(json.RawMessage, sz)
+	copy(ra, a)
+	return newLazyNode(&ra), sz, nil
+}
+
 func (n *lazyNode) intoDoc() (*partialDoc, error) {
 	if n.which == eDoc {
 		return &n.doc, nil
@@ -344,35 +366,14 @@ func (d *partialDoc) remove(key string) error {
 	return nil
 }
 
+// set should only be used to implement the "replace" operation, so "key" must
+// be an already existing index in "d".
 func (d *partialArray) set(key string, val *lazyNode) error {
-	if key == "-" {
-		*d = append(*d, val)
-		return nil
-	}
-
 	idx, err := strconv.Atoi(key)
 	if err != nil {
 		return err
 	}
-
-	sz := len(*d)
-	if idx+1 > sz {
-		sz = idx + 1
-	}
-
-	ary := make([]*lazyNode, sz)
-
-	cur := *d
-
-	copy(ary, cur)
-
-	if idx >= len(ary) {
-		return fmt.Errorf("Unable to access invalid index: %d", idx)
-	}
-
-	ary[idx] = val
-
-	*d = ary
+	(*d)[idx] = val
 	return nil
 }
 
@@ -387,7 +388,9 @@ func (d *partialArray) add(key string, val *lazyNode) error {
 		return err
 	}
 
-	ary := make([]*lazyNode, len(*d)+1)
+	sz := len(*d) + 1
+
+	ary := make([]*lazyNode, sz)
 
 	cur := *d
 
@@ -527,7 +530,7 @@ func (p Patch) move(doc *container, op operation) error {
 		return fmt.Errorf("jsonpatch move operation does not apply: doc is missing destination path: %s", path)
 	}
 
-	return con.set(key, val)
+	return con.add(key, val)
 }
 
 func (p Patch) test(doc *container, op operation) error {
@@ -561,7 +564,7 @@ func (p Patch) test(doc *container, op operation) error {
 	return fmt.Errorf("Testing value %s failed", path)
 }
 
-func (p Patch) copy(doc *container, op operation) error {
+func (p Patch) copy(doc *container, op operation, accumulatedCopySize *int64) error {
 	from := op.from()
 
 	con, key := findObject(doc, from)
@@ -583,7 +586,16 @@ func (p Patch) copy(doc *container, op operation) error {
 		return fmt.Errorf("jsonpatch copy operation does not apply: doc is missing destination path: %s", path)
 	}
 
-	return con.set(key, val)
+	valCopy, sz, err := deepCopy(val)
+	if err != nil {
+		return err
+	}
+	(*accumulatedCopySize) += int64(sz)
+	if AccumulatedCopySizeLimit > 0 && *accumulatedCopySize > AccumulatedCopySizeLimit {
+		return NewAccumulatedCopySizeError(AccumulatedCopySizeLimit, *accumulatedCopySize)
+	}
+
+	return con.add(key, valCopy)
 }
 
 // Equal indicates if 2 JSON documents have the same structural equality.
@@ -636,6 +648,8 @@ func (p Patch) ApplyIndent(doc []byte, indent string) ([]byte, error) {
 
 	err = nil
 
+	var accumulatedCopySize int64
+
 	for _, op := range p {
 		switch op.kind() {
 		case "add":
@@ -649,7 +663,7 @@ func (p Patch) ApplyIndent(doc []byte, indent string) ([]byte, error) {
 		case "test":
 			err = p.test(&pd, op)
 		case "copy":
-			err = p.copy(&pd, op)
+			err = p.copy(&pd, op, &accumulatedCopySize)
 		default:
 			err = fmt.Errorf("Unexpected kind: %s", op.kind())
 		}

vendor/github.com/gogo/protobuf/proto/encode.go

@@ -189,6 +189,8 @@ type Marshaler interface {
 // prefixed by a varint-encoded length.
 func (p *Buffer) EncodeMessage(pb Message) error {
 	siz := Size(pb)
+	sizVar := SizeVarint(uint64(siz))
+	p.grow(siz + sizVar)
 	p.EncodeVarint(uint64(siz))
 	return p.Marshal(pb)
 }

vendor/github.com/gogo/protobuf/proto/properties.go

@@ -43,7 +43,6 @@ package proto
 import (
 	"fmt"
 	"log"
-	"os"
 	"reflect"
 	"sort"
 	"strconv"
@@ -205,7 +204,7 @@ func (p *Properties) Parse(s string) {
 	// "bytes,49,opt,name=foo,def=hello!"
 	fields := strings.Split(s, ",") // breaks def=, but handled below.
 	if len(fields) < 2 {
-		fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s)
+		log.Printf("proto: tag has too few fields: %q", s)
 		return
 	}
 
@@ -225,7 +224,7 @@ func (p *Properties) Parse(s string) {
 		p.WireType = WireBytes
 		// no numeric converter for non-numeric types
 	default:
-		fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s)
+		log.Printf("proto: tag has unknown wire type: %q", s)
 		return
 	}
 

vendor/github.com/gogo/protobuf/proto/table_marshal.go

@@ -2969,7 +2969,9 @@ func (p *Buffer) Marshal(pb Message) error {
 	if m, ok := pb.(newMarshaler); ok {
 		siz := m.XXX_Size()
 		p.grow(siz) // make sure buf has enough capacity
-		p.buf, err = m.XXX_Marshal(p.buf, p.deterministic)
+		pp := p.buf[len(p.buf) : len(p.buf) : len(p.buf)+siz]
+		pp, err = m.XXX_Marshal(pp, p.deterministic)
+		p.buf = append(p.buf, pp...)
 		return err
 	}
 	if m, ok := pb.(Marshaler); ok {

vendor/github.com/gogo/protobuf/proto/text.go

@@ -476,6 +476,8 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 	return nil
 }
 
+var textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
+
 // writeAny writes an arbitrary field.
 func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error {
 	v = reflect.Indirect(v)
@@ -589,8 +591,8 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
 			// mutating this value.
 			v = v.Addr()
 		}
-		if etm, ok := v.Interface().(encoding.TextMarshaler); ok {
-			text, err := etm.MarshalText()
+		if v.Type().Implements(textMarshalerType) {
+			text, err := v.Interface().(encoding.TextMarshaler).MarshalText()
 			if err != nil {
 				return err
 			}