Merge pull request #881 from chrisplo/detect-python3

Detect python3

.github/workflows/build.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.17.x, 1.18.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}

.github/workflows/image-build.yml

@@ -1,8 +1,26 @@
 name: Image build
 on: [pull_request]
 jobs:
+  ep-build-amd64:
+    name: Image build/amd64 LEGACY entrypoint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:ep-latest-amd64
+          file: images/Dockerfile
+
   build-amd64:
-    name: Image build/amd64
+    name: Image build/amd64 daemonized alternative
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
@@ -17,7 +35,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-amd64
-          file: deployments/Dockerfile
+          file: images/Dockerfile.thick
 
   build-arm64:
     name: Image build/arm64
@@ -35,7 +53,25 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-arm64
-          file: deployments/Dockerfile.arm64
+          file: images/Dockerfile.arm64
+
+  build-arm32:
+    name: Image build/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm32
+          file: images/Dockerfile.arm32
 
   build-ppc64le:
     name: Image build/ppc64le
@@ -53,7 +89,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-ppc64le
-          file: deployments/Dockerfile.ppc64le
+          file: images/Dockerfile.ppc64le
 
   build-s390:
     name: Image build/s390x
@@ -71,7 +107,7 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-s390x
-          file: deployments/Dockerfile.s390x
+          file: images/Dockerfile.s390x
 
   build-origin:
     name: Image build/origin
@@ -89,4 +125,4 @@ jobs:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest-origin
-          file: deployments/Dockerfile.openshift
+          file: images/Dockerfile.openshift

.github/workflows/image-push-master.yml

@@ -3,9 +3,6 @@ on:
   push:
     branches:
       - master
-env:
-  REPOSITORY: docker.io/nfvpe/multus
-  REPOSITORY_USER: nfvperobot
 jobs:
   push-amd64:
     name: Image push/amd64
@@ -17,23 +14,34 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:latest-amd64
-            ${{ env.REPOSITORY }}:snapshot-amd64
-          file: deployments/Dockerfile
+            ghcr.io/${{ github.repository }}:latest-amd64
+            ghcr.io/${{ github.repository }}:snapshot-amd64
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:thick-amd64
+          file: images/Dockerfile.thick
 
   push-arm64:
     name: Image push/arm64
@@ -45,23 +53,53 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:latest-arm64
-            ${{ env.REPOSITORY }}:snapshot-arm64
-          file: deployments/Dockerfile.arm64
+            ghcr.io/${{ github.repository }}:latest-arm64
+            ghcr.io/${{ github.repository }}:snapshot-arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-arm32
+            ghcr.io/${{ github.repository }}:snapshot-arm32
+          file: images/Dockerfile.arm32
 
   push-ppc64le:
     name: Image push/ppc64le
@@ -73,23 +111,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:latest-ppc64le
-            ${{ env.REPOSITORY }}:snapshot-ppc64le
-          file: deployments/Dockerfile.ppc64le
+            ghcr.io/${{ github.repository }}:latest-ppc64le
+            ghcr.io/${{ github.repository }}:snapshot-ppc64le
+          file: images/Dockerfile.ppc64le
 
   push-s390x:
     name: Image push/s390x
@@ -101,23 +140,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:latest-s390x
-            ${{ env.REPOSITORY }}:snapshot-s390x
-          file: deployments/Dockerfile.s390x
+            ghcr.io/${{ github.repository }}:latest-s390x
+            ghcr.io/${{ github.repository }}:snapshot-s390x
+          file: images/Dockerfile.s390x
 
   push-origin:
     name: Image push/origin
@@ -129,59 +169,72 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:latest-origin
-            ${{ env.REPOSITORY }}:snapshot-origin
-          file: deployments/Dockerfile.openshift
+            ghcr.io/${{ github.repository }}:latest-origin
+            ghcr.io/${{ github.repository }}:snapshot-origin
+          file: images/Dockerfile.openshift
 
   push-manifest:
     needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
     runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
     steps:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create manifest for multi-arch images
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         run: |
           # get artifacts from previous steps
+          docker pull ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest create ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64 --arch amd64
+          docker manifest push ${{ env.REPOSITORY }}:thick
+
           docker pull ${{ env.REPOSITORY }}:snapshot-amd64
           docker pull ${{ env.REPOSITORY }}:snapshot-arm64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm32
           docker pull ${{ env.REPOSITORY }}:snapshot-ppc64le
           docker pull ${{ env.REPOSITORY }}:snapshot-s390x
           docker pull ${{ env.REPOSITORY }}:latest-amd64
           docker pull ${{ env.REPOSITORY }}:latest-arm64
+          docker pull ${{ env.REPOSITORY }}:latest-arm32
           docker pull ${{ env.REPOSITORY }}:latest-ppc64le
           docker pull ${{ env.REPOSITORY }}:latest-s390x
-          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
+          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-arm32 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
           docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 --arch amd64
           docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm32 --arch arm
           docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-ppc64le --arch ppc64le
           docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-s390x --arch s390x
           docker manifest push ${{ env.REPOSITORY }}:snapshot
-          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-arm32 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
           docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 --arch amd64
           docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm32 --arch arm
           docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-ppc64le --arch ppc64le
           docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-s390x --arch s390x
           docker manifest push ${{ env.REPOSITORY }}:latest

.github/workflows/image-push-release.yml

@@ -3,9 +3,6 @@ on:
   push:
     tags:
       - v*
-env:
-  REPOSITORY: docker.io/nfvpe/multus
-  REPOSITORY_USER: nfvperobot
 jobs:
   push-amd64:
     name: Image push/amd64
@@ -17,30 +14,42 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:stable-amd64
+            ghcr.io/${{ github.repository }}:stable-amd64
             ${{ steps.docker_meta.outputs.tags }}-amd64
-          file: deployments/Dockerfile
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:thick-amd64
+            ${{ steps.docker_meta.outputs.tags }}-thick-amd64
+          file: images/Dockerfile.thick
 
   push-arm64:
     name: Image push/arm64
@@ -52,30 +61,67 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:stable-arm64
+            ghcr.io/${{ github.repository }}:stable-arm64
             ${{ steps.docker_meta.outputs.tags }}-arm64
-          file: deployments/Dockerfile.arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-arm32
+            ${{ steps.docker_meta.outputs.tags }}-arm32
+          file: images/Dockerfile.arm32
 
   push-ppc64le:
     name: Image push/ppc64le
@@ -87,30 +133,31 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:stable-ppc64le
+            ghcr.io/${{ github.repository }}:stable-ppc64le
             ${{ steps.docker_meta.outputs.tags }}-ppc64le
-          file: deployments/Dockerfile.ppc64le
+          file: images/Dockerfile.ppc64le
 
   push-s390x:
     name: Image push/s390x
@@ -122,30 +169,31 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:stable-s390x
+            ghcr.io/${{ github.repository }}:stable-s390x
             ${{ steps.docker_meta.outputs.tags }}-s390x
-          file: deployments/Dockerfile.s390x
+          file: images/Dockerfile.s390x
 
   push-origin:
     name: Image push/origin
@@ -157,34 +205,37 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
       - name: Push container image
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ${{ env.REPOSITORY }}:stable-origin
+            ghcr.io/${{ github.repository }}:stable-origin
             ${{ steps.docker_meta.outputs.tags }}-origin
-          file: deployments/Dockerfile.openshift
+          file: images/Dockerfile.openshift
 
   push-manifest:
     needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
     runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
     steps:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -193,37 +244,42 @@ jobs:
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: ${{ env.REPOSITORY }}
+          images: ghcr.io/${{ github.repository }}
           tag-latest: false
 
-      - name: Login to Container Registry
-        if: github.repository_owner == 'intel'
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         uses: docker/login-action@v1
         with:
-          username: ${{ env.REPOSITORY_USER }}
-          password: ${{ secrets.REPOSITORY_PASS }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create manifest for multi-arch images
-        if: github.repository_owner == 'intel'
+        if: github.repository_owner == 'k8snetworkplumbingwg'
         run: |
           # get artifacts from previous steps
           docker pull ${{ steps.docker_meta.outputs.tags }}-amd64
           docker pull ${{ steps.docker_meta.outputs.tags }}-arm64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm32
           docker pull ${{ steps.docker_meta.outputs.tags }}-ppc64le
           docker pull ${{ steps.docker_meta.outputs.tags }}-s390x
-          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-arm32 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
           docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 --arch amd64
           docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm64 --arch arm64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm32 --arch arm
           docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-ppc64le --arch ppc64le
           docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-s390x --arch s390x
           docker manifest push ${{ steps.docker_meta.outputs.tags }}
           docker pull ${{ env.REPOSITORY }}:stable-amd64
           docker pull ${{ env.REPOSITORY }}:stable-arm64
+          docker pull ${{ env.REPOSITORY }}:stable-arm32
           docker pull ${{ env.REPOSITORY }}:stable-ppc64le
           docker pull ${{ env.REPOSITORY }}:stable-s390x
-          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-arm32 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
           docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 --arch amd64
           docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm32 --arch arm
           docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-ppc64le --arch ppc64le
           docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-s390x --arch s390x
           docker manifest push ${{ env.REPOSITORY }}:stable

.github/workflows/kind-e2e.yml

@@ -14,7 +14,7 @@ jobs:
         run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
   
       - name: Build latest-amd64
-        run: docker build -t localhost:5000/multus:e2e -f deployments/Dockerfile .
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile.thick .
  
       - name: Push to local registry
         run: docker push localhost:5000/multus:e2e
@@ -27,10 +27,18 @@ jobs:
         working-directory: ./e2e
         run: ./setup_cluster.sh
 
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
       - name: Test macvlan1
         working-directory: ./e2e
         run: ./test-simple-macvlan1.sh
 
+      - name: Test static pod
+        working-directory: ./e2e
+        run: ./test-static-pod.sh
+
       - name: Test default route1
         working-directory: ./e2e
         run: ./test-default-route1.sh

.github/workflows/legacy-kind-e2e.yml

@@ -0,0 +1,46 @@
+name: e2e-kind legacy installation with entrypoint script
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile .
+
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: MULTUS_MANIFEST=legacy-multus-daemonset.yml ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15.x
+          go-version: 1.17.x
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2

.github/workflows/stale-issues-prs.yml

@@ -0,0 +1,15 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v3
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          days-before-stale: 90
+          days-before-close: 7

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:

.goreleaser.yml

@@ -18,6 +18,8 @@ builds:
   - arm
   - arm64
   - s390x
+  ldflags:
+  - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date={{ .Date }}
 archives:
   - wrap_in_directory: true
 checksum:

README.md

@@ -1,8 +1,8 @@
 # Multus-CNI
 
-![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/docs/images/Multus.png)
+![multus-cni Logo](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/images/Multus.png)
 
-[![Build](https://github.com/intel/multus-cni/actions/workflows/build.yml/badge.svg)](https://github.com/intel/multus-cni/actions/workflows/build.yml)[![Test](https://github.com/intel/multus-cni/actions/workflows/test.yml/badge.svg)](https://github.com/intel/multus-cni/actions/workflows/test.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/intel/multus-cni/badge.svg)](https://coveralls.io/github/intel/multus-cni)
+[![Build](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml)[![Test](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/k8snetworkplumbingwg/multus-cni)](https://goreportcard.com/report/github.com/k8snetworkplumbingwg/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/k8snetworkplumbingwg/multus-cni/badge.svg)](https://coveralls.io/github/k8snetworkplumbingwg/multus-cni)
 
 Multus CNI enables attaching multiple network interfaces to pods in Kubernetes.
 
@@ -22,12 +22,12 @@ Here's an illustration of the network interfaces attached to a pod, as provision
 
 ## Quickstart Installation Guide
 
-The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be creatd with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
 
 Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
@@ -35,7 +35,7 @@ This will configure your systems to be ready to use Multus CNI, but, to get star
 ## Additional installation Options
 
 - Install via daemonset using the quick-start guide, above.
-- Download binaries from [release page](https://github.com/intel/multus-cni/releases)
+- Download binaries from [release page](https://github.com/k8snetworkplumbingwg/multus-cni/releases)
 - By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
 - Or, roll-your-own and build from source
   - See [Development](docs/development.md)

cmd/config-generation/kubeconfig.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this generates kubeconfig file for multus based on service account
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+const userRWPermission = 0600
+
+const (
+	cniConfigDirVarName   = "cni-config-dir"
+	k8sCAFilePathVarName  = "kube-ca-file"
+	k8sServiceHostVarName = "k8s-service-host"
+	k8sServicePortVarName = "k8s-service-port"
+	serviceAccountPath    = "/var/run/secrets/kubernetes.io/serviceaccount"
+	skipTLSVerifyVarName  = "skip-tls-verify"
+)
+
+const (
+	defaultCniConfigDir   = "/host/etc/cni/net.d"
+	defaultK8sCAFilePath  = ""
+	defaultK8sServiceHost = ""
+	defaultK8sServicePort = 0
+	defaultSkipTLSValue   = false
+)
+
+func main() {
+	k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
+	k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
+	skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
+	kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	flag.Parse()
+
+	if *k8sServiceHost == defaultK8sServiceHost {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *k8sServicePort == defaultK8sServicePort {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *kubeCAFilePath == defaultK8sServiceHost {
+		*kubeCAFilePath = serviceAccountPath + "/ca.crt"
+	}
+
+	tlsCfg := "insecure-skip-tls-verify: true"
+	if !*skipTLSVerify {
+		kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
+		if err != nil {
+			logError("failed grabbing CA file: %w", err)
+		}
+		tlsCfg = "certificate-authority-data: " + kubeCAFileContents
+	}
+
+	multusConfigDir := *cniConfigDir + "/multus.d/"
+	if err := prepareCNIConfigDir(multusConfigDir); err != nil {
+		logError("failed to create CNI config dir: %w", err)
+	}
+	kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
+	serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
+	if err != nil {
+		logError("failed grabbing k8s token: %w", err)
+	}
+	if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
+		logError("failed generating kubeconfig: %w", err)
+	}
+}
+
+func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
+	data, err := ioutil.ReadFile(pathCAFile)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
+	}
+	return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
+}
+
+func k8sKubeConfigToken(tokenPath string) (string, error) {
+	data, err := ioutil.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
+	}
+	return string(data), nil
+}
+
+func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
+	kubeConfigTemplate := `
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: %s://[%s]:%d
+    %s
+users:
+- name: multus
+  user:
+    token: "%s"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+	kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
+	logInfo("Generated KubeConfig saved to %s: \n%s", outputPath, kubeconfig)
+	return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
+}
+
+func prepareCNIConfigDir(cniConfigDirPath string) error {
+	return os.MkdirAll(cniConfigDirPath, userRWPermission)
+}
+
+func logInvalidArg(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func logError(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	os.Exit(1)
+}
+
+func logInfo(format string, values ...interface{}) {
+	log.Printf("INFO: %s", fmt.Sprintf(format, values...))
+}

cmd/controller/main.go

@@ -0,0 +1,253 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this is daemonized entrypoint process. which watches master config
+// and generate multus CNI config
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+const (
+	multusPluginName     = "multus"
+	multusConfigFileName = "00-multus.conf"
+)
+
+const (
+	defaultCniConfigDir                 = "/etc/cni/net.d"
+	defaultMultusAdditionalBinDir       = ""
+	defaultMultusCNIVersion             = ""
+	defaultMultusConfigFile             = "auto"
+	defaultMultusGlobalNamespaces       = ""
+	defaultMultusKubeconfigPath         = "/etc/cni/net.d/multus.d/multus.kubeconfig"
+	defaultMultusLogFile                = ""
+	defaultMultusLogMaxSize             = 100 // megabytes
+	defaultMultusLogMaxAge              = 5   // days
+	defaultMultusLogMaxBackups          = 5
+	defaultMultusLogCompress            = true
+	defaultMultusLogLevel               = ""
+	defaultMultusLogToStdErr            = false
+	defaultMultusMasterCNIFile          = ""
+	defaultMultusNamespaceIsolation     = false
+	defaultMultusReadinessIndicatorFile = ""
+)
+
+const (
+	cniConfigDirVarName           = "cni-config-dir"
+	multusAdditionalBinDirVarName = "additional-bin-dir"
+	multusAutoconfigDirVarName    = "multus-autoconfig-dir"
+	multusCNIVersion              = "cni-version"
+	multusConfigFileVarName       = "multus-conf-file"
+	multusGlobalNamespaces        = "global-namespaces"
+	multusLogFile                 = "multus-log-file"
+	multusLogMaxSize              = "multus-log-max-size"
+	multusLogMaxAge               = "multus-log-max-age"
+	multusLogMaxBackups           = "multus-log-max-backups"
+	multusLogCompress             = "multus-log-compress"
+	multusLogLevel                = "multus-log-level"
+	multusLogToStdErr             = "multus-log-to-stderr"
+	multusKubeconfigPath          = "multus-kubeconfig-file-host"
+	multusMasterCNIFileVarName    = "multus-master-cni-file"
+	multusNamespaceIsolation      = "namespace-isolation"
+	multusReadinessIndicatorFile  = "readiness-indicator-file"
+)
+
+func main() {
+	versionOpt := false
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	multusConfigFile := flag.String(multusConfigFileVarName, defaultMultusConfigFile, "The multus configuration file to use. By default, a new configuration is generated.")
+	multusMasterCni := flag.String(multusMasterCNIFileVarName, defaultMultusMasterCNIFile, "The relative name of the configuration file of the cluster primary CNI.")
+	multusAutoconfigDir := flag.String(multusAutoconfigDirVarName, *cniConfigDir, "The directory path for the generated multus configuration.")
+	namespaceIsolation := flag.Bool(multusNamespaceIsolation, defaultMultusNamespaceIsolation, "If the network resources are only available within their defined namespaces.")
+	globalNamespaces := flag.String(multusGlobalNamespaces, defaultMultusGlobalNamespaces, "Comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled.")
+	logToStdErr := flag.Bool(multusLogToStdErr, defaultMultusLogToStdErr, "If the multus logs are also to be echoed to stderr.")
+	logLevel := flag.String(multusLogLevel, defaultMultusLogLevel, "One of: debug/verbose/error/panic. Used only with --multus-conf-file=auto.")
+	logFile := flag.String(multusLogFile, defaultMultusLogFile, "Path where to multus will log. Used only with --multus-conf-file=auto.")
+	logMaxSize := flag.Int(multusLogMaxSize, defaultMultusLogMaxSize, "the maximum size in megabytes of the log file before it gets rotated")
+	logMaxAge := flag.Int(multusLogMaxAge, defaultMultusLogMaxAge, "the maximum number of days to retain old log files in their filename")
+	logMaxBackups := flag.Int(multusLogMaxBackups, defaultMultusLogMaxBackups, "the maximum number of old log files to retain")
+	logCompress := flag.Bool(multusLogCompress, defaultMultusLogCompress, "compress determines if the rotated log files should be compressed using gzip")
+	cniVersion := flag.String(multusCNIVersion, defaultMultusCNIVersion, "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
+	additionalBinDir := flag.String(multusAdditionalBinDirVarName, defaultMultusAdditionalBinDir, "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
+	readinessIndicator := flag.String(multusReadinessIndicatorFile, defaultMultusReadinessIndicatorFile, "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
+	multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
+	overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	if *logToStdErr {
+		logging.SetLogStderr(*logToStdErr)
+	}
+	if *logFile != defaultMultusLogFile {
+		logging.SetLogFile(*logFile)
+	}
+	if *logLevel != defaultMultusLogLevel {
+		logging.SetLogLevel(*logLevel)
+	}
+
+	if *multusConfigFile == defaultMultusConfigFile {
+		if *cniVersion == defaultMultusCNIVersion {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		var configurationOptions []config.Option
+		if *namespaceIsolation {
+			configurationOptions = append(
+				configurationOptions, config.WithNamespaceIsolation())
+		}
+
+		if *globalNamespaces != defaultMultusGlobalNamespaces {
+			configurationOptions = append(
+				configurationOptions, config.WithGlobalNamespaces(*globalNamespaces))
+		}
+
+		if *logToStdErr != defaultMultusLogToStdErr {
+			configurationOptions = append(
+				configurationOptions, config.WithLogToStdErr())
+		}
+
+		if *logLevel != defaultMultusLogLevel {
+			configurationOptions = append(
+				configurationOptions, config.WithLogLevel(*logLevel))
+		}
+
+		if *logFile != defaultMultusLogFile {
+			configurationOptions = append(
+				configurationOptions, config.WithLogFile(*logFile))
+		}
+
+		if *additionalBinDir != defaultMultusAdditionalBinDir {
+			configurationOptions = append(
+				configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
+		}
+
+		if *readinessIndicator != defaultMultusReadinessIndicatorFile {
+			configurationOptions = append(
+				configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
+		}
+
+		// logOptions
+
+		var logOptionFuncs []config.LogOptionFunc
+		if *logMaxAge != defaultMultusLogMaxAge {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxAge(logMaxAge))
+		}
+		if *logMaxSize != defaultMultusLogMaxSize {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxSize(logMaxSize))
+		}
+		if *logMaxBackups != defaultMultusLogMaxBackups {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxBackups(logMaxBackups))
+		}
+		if *logCompress != defaultMultusLogCompress {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogCompress(logCompress))
+		}
+
+		if len(logOptionFuncs) > 0 {
+			logOptions := &config.LogOptions{}
+			config.MutateLogOptions(logOptions, logOptionFuncs...)
+			configurationOptions = append(configurationOptions, config.WithLogOptions(logOptions))
+		}
+
+		multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
+		if err != nil {
+			_ = logging.Errorf("Failed to create multus config: %v", err)
+			os.Exit(3)
+		}
+
+		var configManager *config.Manager
+		if *multusMasterCni == "" {
+			configManager, err = config.NewManager(*multusConfig, *multusAutoconfigDir)
+		} else {
+			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
+				*multusConfig, *multusAutoconfigDir, *multusMasterCni)
+		}
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+
+		if *overrideNetworkName {
+			if err := configManager.OverrideNetworkName(); err != nil {
+				_ = logging.Errorf("could not override the network name: %v", err)
+			}
+		}
+
+		generatedMultusConfig, err := configManager.GenerateConfig()
+		if err != nil {
+			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
+		}
+		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
+			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+		}
+
+		configWatcherDoneChannel := make(chan struct{})
+		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
+			defer func() {
+				stopChannel <- struct{}{}
+			}()
+			if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
+				_ = logging.Errorf("error watching file: %v", err)
+			}
+		}(make(chan struct{}), configWatcherDoneChannel)
+
+		<-configWatcherDoneChannel
+	} else {
+		if err := copyUserProvidedConfig(*multusConfigFile, *cniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", *multusConfigFile, err)
+		}
+	}
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	srcFile, err := os.Open(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(srcFile, dstFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/main.go

@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This is a "Multi-plugin".The delegate concept refered from CNI project
+// This is a "Multi-plugin".The delegate concept referred from CNI project
 // It reads other plugin netconf, and then invoke them, e.g.
 // flannel or sriov plugin.
-
 package main
 
 import (
@@ -23,12 +22,11 @@ import (
 	"fmt"
 	"os"
 
-	"gopkg.in/intel/multus-cni.v3/pkg/multus"
 	"github.com/containernetworking/cni/pkg/skel"
 	cniversion "github.com/containernetworking/cni/pkg/version"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
 )
 
-
 func main() {
 
 	// Init command line flags to clear vendored packages' one, especially in init()

deployments/Dockerfile

@@ -1,20 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM centos:centos7 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV INSTALL_PKGS "git golang-1.13.10-0.el7.x86_64"
-RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
-    curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo && \
-    yum install -y $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    cd /usr/src/multus-cni && \
-    ./hack/build-go.sh
-
-FROM centos:centos7
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
-WORKDIR /
-
-ADD ./images/entrypoint.sh /
-ENTRYPOINT ["/entrypoint.sh"]

deployments/Dockerfile.ppc64le

@@ -1,25 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM centos:centos7 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV GOARCH "ppc64le"
-ENV GOOS "linux"
-
-ENV INSTALL_PKGS "git golang"
-RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
-    curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo && \
-    yum install -y $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    cd /usr/src/multus-cni && \
-    ./hack/build-go.sh
-
-# build ppc container
-FROM ppc64le/centos:latest
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
-
-WORKDIR /
-ADD ./images/entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]

deployments/multus-daemonset-crio.yml

@@ -143,7 +143,7 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-multus-ds-amd64
+  name: kube-multus-ds
   namespace: kube-system
   labels:
     tier: node
@@ -163,16 +163,16 @@ spec:
         name: multus
     spec:
       hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
       serviceAccountName: multus
       containers:
       - name: kube-multus
         # crio support requires multus:latest for now. support 3.3 or later.
-        image: docker.io/nfvpe/multus:stable
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
         command: ["/entrypoint.sh"]
         args:
         - "--cni-version=0.3.1"
@@ -193,6 +193,7 @@ spec:
         volumeMounts:
         - name: run
           mountPath: /run
+          mountPropagation: HostToContainer
         - name: cni
           mountPath: /host/etc/cni/net.d
         - name: cnibin
@@ -216,213 +217,3 @@ spec:
             items:
             - key: cni-conf.json
               path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: docker.io/nfvpe/multus:stable-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-version=0.3.1"
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-arm64v8
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: arm64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: docker.io/nfvpe/multus:stable-arm64v8
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-version=0.3.1"
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-s390x
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: s390x
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: docker.io/nfvpe/multus:stable-s390x
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-version=0.3.1"
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

deployments/multus-daemonset-gke-1.16.yml

@@ -116,7 +116,7 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-multus-ds-amd64
+  name: kube-multus-ds
   namespace: kube-system
   labels:
     tier: node
@@ -136,15 +136,15 @@ spec:
         name: multus
     spec:
       hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
       tolerations:
         - operator: Exists
           effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
       serviceAccountName: multus
       containers:
         - name: kube-multus
-          image: docker.io/nfvpe/multus:stable
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
           command: ["/entrypoint.sh"]
           args:
             - "--multus-conf-file=auto"
@@ -179,71 +179,3 @@ spec:
             items:
               - key: cni-conf.json
                 path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: ppc64le
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          # ppc64le support requires multus:latest for now. support 3.3 or later.
-          image: docker.io/nfvpe/multus:stable-ppc64le
-          command: ["/entrypoint.sh"]
-          args:
-            - "--multus-conf-file=auto"
-            - "--cni-version=0.3.1"
-            - "--cni-bin-dir=/host/home/kubernetes/bin"
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "90Mi"
-            limits:
-              cpu: "100m"
-              memory: "90Mi"
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /host/home/kubernetes/bin
-            - name: multus-cfg
-              mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /home/kubernetes/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-              - key: cni-conf.json
-                path: 70-multus.conf

deployments/multus-daemonset-thick-plugin.yml

@@ -0,0 +1,189 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
+          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+          args:
+            - "-cni-version=0.3.1"
+            - "-cni-config-dir=/host/etc/cni/net.d"
+            - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+            - "-multus-log-to-stderr=true"
+            - "-multus-log-level=verbose"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+        - name: generate-kubeconfig
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
+          command:
+            - "/usr/src/multus-cni/bin/generate-kubeconfig"
+          args:
+            - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+            - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+

deployments/multus-daemonset.yml

@@ -0,0 +1,225 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

docs/configuration.md

@@ -5,6 +5,7 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
 
 ```
 {
+    "cniVersion": "0.3.1",
     "name": "node-cni-network",
     "type": "multus",
     "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
@@ -13,6 +14,12 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     "binDir": "/opt/cni/bin",
     "logFile": "/var/log/multus.log",
     "logLevel": "debug",
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    },
     "capabilities": {
         "portMappings": true
     },    
@@ -39,18 +46,19 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
 * `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
 * `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
 * `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
-* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
 * `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
 * `logFile` (string, optional): file path for log file. multus puts log in given file
 * `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
+* `logOptions` (object, optional): logging option, More detailed log configuration
 * `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
-* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf).
-* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
 
 User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
 
-* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
-* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
+* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
 * `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
 * `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
 * `delegates` ([]map,required): number of delegate details in the Multus
@@ -60,7 +68,7 @@ User should chose following parameters combination (`clusterNetwork`+`defaultNet
 Multus will find network for clusterNetwork/defaultNetworks as following sequences:
 
 1. CRD object for given network name, in 'kube-system' namespace
-1. CNI json config file in `confDir`. Given name should be without extention, like .conf/.conflist. (e.g. "test" for "test.conf")
+1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
 1. Directory for CNI json config file. Multus will find alphabetically first file for the network
 1. Multus failed to find network. Multus raise error message
 
@@ -76,7 +84,7 @@ In this manner, you may prevent pods from crash looping, and instead wait for th
 
 Only one option is necessary to configure this functionality:
 
-* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready.
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready.
 
 *NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
 
@@ -102,7 +110,7 @@ Optionally, you may have Multus log to a file on the filesystem. This file will
 For example in your CNI configuration, you may set:
 
 ```
-    "LogFile": "/var/log/multus.log",
+    "logFile": "/var/log/multus.log",
 ```
 
 #### Logging Level
@@ -119,7 +127,27 @@ The available logging level values, in decreasing order of verbosity are:
 You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
 
 ```
-    "LogLevel": "debug",
+    "logLevel": "debug",
+```
+
+#### Logging Options
+
+If you want a more detailed configuration of the logging, This includes the following parameters:
+
+* `maxAge` the maximum number of days to retain old log files in their filename
+* `maxSize` the maximum size in megabytes of the log file before it gets rotated
+* `maxBackups` the maximum number of days to retain old log files in their filename
+* `compress` compress determines if the rotated log files should be compressed using gzip
+
+For example in your CNI configuration, you may set:
+
+```
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    }
 ```
 
 ### Namespace Isolation

docs/development.md

@@ -6,7 +6,7 @@ Multus now uses [gopkg.in](http://gopkg.in/) to expose its code as library.
 You can use following command to import our code into your go code.
 
 ```
-go get gopkg.in/intel/multus-cni.v3
+go get gopkg.in/k8snetworkplumbingwg/multus-cni.v3
 ```
 
 
@@ -25,7 +25,7 @@ If an issue is closed that you don't feel is sufficiently resolved, please feel
 You can use the built in `./hack/build-go.sh` script!
 
 ```
-git clone https://github.com/intel/multus-cni.git
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 cd multus-cni
 ./hack/build-go.sh
 ```
@@ -35,18 +35,18 @@ cd multus-cni
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
 ```
-sudo ./scripts/test.sh
+sudo ./hack/test-go.sh
 ```
 
 ## What are the best practices for logging?
 
 The following are the best practices for multus logging:
 
-* Add `logging.Debugf()` at the begining of functions
+* Add `logging.Debugf()` at the beginning of functions
 * In case of error handling, use `logging.Errorf()` with given error info
 * `logging.Panicf()` only be used for critical errors (it should NOT normally be used)
 
 
-## CI Introduction
+## Multus release schedule
 
-TBD
+On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.

docs/how-to-use.md

@@ -13,15 +13,15 @@ Generally we recommend two options: Manually place a Multus binary in your `/opt
 
 *Copy Multus Binary into place*
 
-You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/intel/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
+You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/k8snetworkplumbingwg/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
 
-    $ cp multus /opt/cni/bin
+    cp multus /opt/cni/bin
 
 *Via Daemonset method*
 
 As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    $ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
+    cat ./deployments/multus-daemonset.yml | kubectl apply -f -
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -34,8 +34,8 @@ You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alp
 Execute following commands at all Kubernetes nodes (i.e. master and minions)
 
 ```
-$ mkdir -p /etc/cni/net.d
-$ cat >/etc/cni/net.d/30-multus.conf <<EOF
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/00-multus.conf <<EOF
 {
   "name": "multus-cni-network",
   "type": "multus",
@@ -72,7 +72,7 @@ Create resources for multus to access CRD objects as following command:
 
 ```
 # Execute following commands at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -119,13 +119,13 @@ Create kubeconfig at master node as following commands:
 
 ```
 # Execute following command at Kubernetes master
-$ mkdir -p /etc/cni/net.d/multus.d
-$ SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
-$ SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-$ KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
-$ KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
-$ KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
-$ cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
+mkdir -p /etc/cni/net.d/multus.d
+SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
+SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
+KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
+KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
+cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
 # Kubeconfig file for Multus CNI plugin.
 apiVersion: v1
 kind: Config
@@ -151,7 +151,7 @@ Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
 **NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
 
 ```
-$ scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
+scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
 ```
 
 ### Setup CRDs (daemonset automatically does)
@@ -162,7 +162,7 @@ Create CRD definition in Kubernetes as following command at master node:
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -200,7 +200,7 @@ Following command creates NetworkAttachmentDefinition. CNI config is in `config:
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -232,7 +232,7 @@ If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -242,7 +242,7 @@ EOF
 
 ```
 # Execute following commands at all Kubernetes nodes (i.e. master and minions)
-$ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
+cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
 {
   "cniVersion": "0.3.0",
   "type": "macvlan",
@@ -264,11 +264,11 @@ $ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
 
 ### Run pod with network annotation
 
-#### Lauch pod with text annotation
+#### Launch pod with text annotation
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -284,13 +284,13 @@ spec:
 EOF
 ```
 
-#### Lauch pod with text annotation for NetworkAttachmentDefinition in different namespace
+#### Launch pod with text annotation for NetworkAttachmentDefinition in different namespace
 
 You can also specify NetworkAttachmentDefinition with its namespace as adding `<namespace>/`
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -314,7 +314,7 @@ spec:
             }
         }'
 EOF
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -330,13 +330,13 @@ spec:
 EOF
 ```
 
-#### Lauch pod with text annotation with interface name
+#### Launch pod with text annotation with interface name
 
 You can also specify interface name as adding `@<ifname>`.
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -352,11 +352,11 @@ spec:
 EOF
 ```
 
-#### Lauch pod with json annotation
+#### Launch pod with json annotation
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -375,13 +375,13 @@ spec:
 EOF
 ```
 
-#### Lauch pod with json annotation for NetworkAttachmentDefinition in different namespace
+#### Launch pod with json annotation for NetworkAttachmentDefinition in different namespace
 
 You can also specify NetworkAttachmentDefinition with its namespace as adding `"namespace": "<namespace>"`.
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -400,13 +400,13 @@ spec:
 EOF
 ```
 
-#### Lauch pod with json annotation with interface
+#### Launch pod with json annotation with interface
 
 You can also specify interface name as adding `"interface": "<ifname>"`.
 
 ```
 # Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
+cat <<EOF | kubectl create -f -
 apiVersion: v1
 kind: Pod
 metadata:
@@ -432,7 +432,8 @@ Following the example of `ip -d address` output of above pod, "pod-case-06":
 
 ```
 # Execute following command at Kubernetes master
-$ kubectl exec -it pod-case-06 -- ip -d address
+kubectl exec -it pod-case-06 -- ip -d address
+
 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
     inet 127.0.0.1/8 scope host lo
@@ -529,7 +530,8 @@ EOF
 This will set `192.168.2.1` as the default route over the `net1` interface, such as:
 
 ```
-$ kubectl exec -it samplepod -- ip route
+kubectl exec -it samplepod -- ip route
+
 default via 192.168.2.1 dev net1 
 10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
 10.244.0.0/16 via 10.244.0.1 dev eth0 
@@ -595,16 +597,20 @@ This can be used if you have your CNI configuration stored in an alternate locat
 
 Used only with `--multus-conf-file=auto`. Allows you to specify an alternate path to the Kubeconfig.
 
+    --multus-master-cni-file-name=
+
+The `--multus-master-cni-file-name` can be used to select the cni file as the master cni, rather than the first file in cni-conf-dir. For example, `--multus-master-cni-file-name=10-calico.conflist`.
+
     --multus-log-level=
     --multus-log-file=
 
-Used only with `--multus-conf-file=auto`. See the documentation for logging for which values are permitted.
+Used only with `--multus-conf-file=auto`. See the [documentation for logging](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#logging) for which values are permitted.
 
-Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to speicfy CNI spec version.
+Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to specify CNI spec version.
 
     --cni-version=
 
-In some cases, the original CNI configuration that the Multus configuration was generated from (using `--multus-conf-file=auto`) may be used as a sort of semaphor for network readiness -- as this model is used by the Kubelet itself. If you need to disable Multus' availablity, you may wish to clean out the generated configuration file when the source file for autogeneration of the config file is no longer present. You can use this functionality by setting:
+In some cases, the original CNI configuration that the Multus configuration was generated from (using `--multus-conf-file=auto`) may be used as a sort of semaphor for network readiness -- as this model is used by the Kubelet itself. If you need to disable Multus' availability, you may wish to clean out the generated configuration file when the source file for autogeneration of the config file is no longer present. You can use this functionality by setting:
 
     --cleanup-config-on-exit=true
 

docs/quickstart.md

@@ -15,7 +15,7 @@ Two things we'll refer to a number of times through this document are:
 
 Our installation method requires that you first have installed Kubernetes and have configured a default network -- that is, a CNI plugin that's used for your pod-to-pod connectivity. 
 
-We recommend Kubernetes 1.16 or later.
+We support Kubernetes versions that Kubernetes community supports. Please see [Supported versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions) in Kubernetes document.
 
 To install Kubernetes, you may decide to use [kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/), or potentially [kubespray](https://github.com/kubernetes-sigs/kubespray).
 
@@ -45,19 +45,19 @@ Our recommended quickstart method to deploy Multus is to deploy using a Daemonse
 Firstly, clone this GitHub repository. 
 
 ```
-git clone https://github.com/intel/multus-cni.git && cd multus-cni
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
 ```
 
 We'll apply a YAML file with `kubectl` from this repo.
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 ### What the Multus daemonset does
 
 * Starts a Multus daemonset, this runs a pod on each node which places a Multus binary on each node in `/opt/cni/bin`
-* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
+* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus on each node as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
 * Creates a `/etc/cni/net.d/multus.d` directory on each node with authentication information for Multus to access the Kubernetes API.
 
 
@@ -66,7 +66,7 @@ $ cat ./images/multus-daemonset.yml | kubectl apply -f -
 Generally, the first step in validating your installation is to ensure that the Multus pods have run without error, you may see an overview of those by looking at:
 
 ```
-$ kubectl get pods --all-namespaces | grep -i multus
+kubectl get pods --all-namespaces | grep -i multus
 ```
 
 You may further validate that it has ran by looking at the `/etc/cni/net.d/` directory and ensure that the auto-generated `/etc/cni/net.d/00-multus.conf` exists corresponding to the alphabetically first configuration file.
@@ -176,7 +176,7 @@ EOF
 You may now inspect the pod and see what interfaces are attached, like so:
 
 ```
-$ kubectl exec -it samplepod -- ip a
+kubectl exec -it samplepod -- ip a
 ```
 
 You should note that there are 3 interfaces:
@@ -191,7 +191,7 @@ For additional confirmation, use `kubectl describe pod samplepod` and there will
 
 ```
 Annotations:        k8s.v1.cni.cncf.io/networks: macvlan-conf
-                    k8s.v1.cni.cncf.io/networks-status:
+                    k8s.v1.cni.cncf.io/network-status:
                       [{
                           "name": "cbr0",
                           "ips": [

e2e/README.md

@@ -4,7 +4,7 @@
 
 
 ```
-$ git clone https://github.com/intel/multus-cni.git
+$ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 $ cd multus-cni/e2e
 $ ./get_tools.sh
 $ ./setup_cluster.sh

e2e/cni-install.yml

@@ -7,9 +7,9 @@ metadata:
 data:
   install_cni.sh: |
     cd /tmp
-    wget https://github.com/containernetworking/plugins/releases/download/v0.8.5/cni-plugins-linux-amd64-v0.8.5.tgz
+    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
     cd /host/opt/cni/bin
-    tar xvfzp /tmp/cni-plugins-linux-amd64-v0.8.5.tgz
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
     sleep infinite
 ---
 apiVersion: apps/v1

e2e/get_tools.sh

@@ -5,11 +5,11 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.8.1/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl
-curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.82/koko_0.82_linux_amd64
+curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq

e2e/legacy-multus-daemonset.yml

@@ -18,30 +18,12 @@ spec:
       storage: true
       schema:
         openAPIV3Schema:
-          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
-            Working Group to express the intent for attaching pods to one or more logical or physical
-            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
           type: object
           properties:
-            apiVersion:
-              description: 'APIVersion defines the versioned schema of this represen
-                tation of an object. Servers should convert recognized schemas to the
-                latest internal value, and may reject unrecognized values. More info:
-                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
-            kind:
-              description: 'Kind is a string value representing the REST resource this
-                object represents. Servers may infer this from the endpoint the client
-                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
-            metadata:
-              type: object
             spec:
-              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
               type: object
               properties:
                 config:
-                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
                   type: string
 ---
 kind: ClusterRole
@@ -62,15 +44,6 @@ rules:
     verbs:
       - get
       - update
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -171,7 +144,7 @@ spec:
       serviceAccountName: multus
       containers:
       - name: kube-multus
-        image: docker.io/nfvpe/multus:stable
+        image: localhost:5000/multus:e2e
         command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"
@@ -192,7 +165,23 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
       volumes:
         - name: cni
           hostPath:
@@ -239,7 +228,7 @@ spec:
       containers:
       - name: kube-multus
         # ppc64le support requires multus:latest for now. support 3.3 or later.
-        image: docker.io/nfvpe/multus:stable-ppc64le
+        image: nfvpe/multus:latest-ppc64le
         command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"
@@ -260,141 +249,6 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-arm64v8
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: arm64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: docker.io/nfvpe/multus:stable-arm64v8
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-s390x
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: s390x
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: docker.io/nfvpe/multus:stable-s390x
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      terminationGracePeriodSeconds: 10
       volumes:
         - name: cni
           hostPath:

e2e/multus-daemonset.yml

@@ -44,6 +44,15 @@ rules:
     verbs:
       - get
       - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -145,10 +154,16 @@ spec:
       containers:
       - name: kube-multus
         image: localhost:5000/multus:e2e
-        command: ["/entrypoint.sh"]
+        imagePullPolicy: Always
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
         args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
+        - "-multus-conf-file=auto"
+        - "-cni-version=0.3.1"
+        - "-cni-config-dir=/host/etc/cni/net.d"
+        - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "-multus-log-to-stderr=true"
+        - "-multus-log-level=debug"
+        - "-multus-log-file=/tmp/multus.log"
         resources:
           requests:
             cpu: "100m"
@@ -165,6 +180,40 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: multus-cfg
           mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      - name: generate-kubeconfig
+        image: localhost:5000/multus:e2e
+        command:
+          - "/usr/src/multus-cni/bin/generate-kubeconfig"
+        args:
+          - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+          - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cni
+            mountPath: /host/etc/cni/net.d
+            mountPropagation: Bidirectional
       volumes:
         - name: cni
           hostPath:

e2e/setup_cluster.sh

@@ -3,19 +3,28 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
+# define the OCI binary to be used. Acceptable values are `docker`, `podman`.
+# Defaults to `docker`.
+OCI_BIN="${OCI_BIN:-docker}"
+
+# define the deployment spec to use when deploying multus.
+# Acceptable values are `legacy-multus-daemonset.yml`. `multus-daemonset.yml`.
+# Defaults to `multus-daemonset.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset.yml}"
+
 kind_network='kind'
 reg_name='kind-registry'
 reg_port='5000'
-running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
 if [ "${running}" != 'true' ]; then
   # run registry and push the multus image
-  docker run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
-  docker build -t localhost:5000/multus:e2e ..
-  docker push localhost:5000/multus:e2e
+  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
+  $OCI_BIN push localhost:5000/multus:e2e
 fi
 reg_host="${reg_name}"
 if [ "${kind_network}" = "bridge" ]; then
-    reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
 fi
 echo "Registry Host: ${reg_host}"
 
@@ -30,6 +39,12 @@ containerdConfigPatches:
 nodes:
   - role: control-plane
   - role: worker
+    kubeadmConfigPatches:
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          pod-manifest-path: "/etc/kubernetes/manifests/"
   - role: worker
 EOF
 
@@ -45,7 +60,7 @@ data:
     help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
 EOF
 
-containers=$(docker network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
+containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
 needs_connect="true"
 for c in $containers; do
   if [ "$c" = "${reg_name}" ]; then
@@ -53,14 +68,17 @@ for c in $containers; do
   fi
 done
 if [ "${needs_connect}" = "true" ]; then
-  docker network connect "${kind_network}" "${reg_name}" || true
+  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
 fi
 
+worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
+worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
+
 kind export kubeconfig
-sudo env PATH=${PATH} koko -d kind-worker,eth1 -d kind-worker2,eth1
+sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
 sleep 1
 kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f multus-daemonset.yml
+kubectl create -f "$MULTUS_MANIFEST"
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
 kubectl create -f cni-install.yml

e2e/simple-pod.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: simple-centos1
+  annotations:
+  labels:
+    app: simple
+spec:
+  containers:
+  - name: simple-centos1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/simple-static-pod.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web
+  annotations:
+    k8s.v1.cni.cncf.io/networks: "bridge-nad"
+spec: 
+  containers: 
+    - name: web 
+      image: centos:8 
+      command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]

e2e/static-pod-nad.yml

@@ -0,0 +1,15 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: bridge-nad
+spec: 
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "testnet",
+    "type": "bridge",
+    "bridge": "testnet0",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "10.10.0.0/16"
+    }
+}'

e2e/test-simple-pod.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f simple-pod.yml
+kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
+
+echo "cleanup resources"
+kubectl delete -f simple-pod.yml

e2e/test-static-pod.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o errexit
+
+echo "Creating network attachment definition"
+kubectl create -f static-pod-nad.yml
+
+echo "Creating static pod config file"
+docker cp simple-static-pod.yml kind-worker:/etc/kubernetes/manifests/static-web.yaml
+
+echo "Waiting for static pod to start"
+kubectl wait --for=condition=Ready --namespace=default pod/static-web-kind-worker
+
+echo "Checking the pod annotation for net1 interface"
+kubectl exec static-web-kind-worker --namespace=default -- ip a show dev net1
+
+echo "Deleting static pod"
+docker exec kind-worker /bin/bash -c "rm /etc/kubernetes/manifests/static-web.yaml"
+
+echo "Deleting network attachment definition"
+kubectl delete -f static-pod-nad.yml
+
+echo "Test complete"

examples/README.md

@@ -62,9 +62,9 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
 
 ## Passing down device information
-Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotation in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
 
-In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+In this example (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
 
 ```yaml
 apiVersion: "k8s.cni.cncf.io/v1"
@@ -89,6 +89,6 @@ spec:
   }
 }'
 ```
-The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition.
+The [sriov-pod.yml](./sriov-pod.yml) is an example Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachment definition.
 
->For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

go.mod

@@ -1,26 +1,53 @@
-module gopkg.in/intel/multus-cni.v3
+module gopkg.in/k8snetworkplumbingwg/multus-cni.v3
 
-go 1.12
+go 1.16
 
 require (
-	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/containernetworking/cni v0.7.1
-	github.com/containernetworking/plugins v0.8.2
-	github.com/json-iterator/go v1.1.9 // indirect
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d
-	github.com/onsi/ginkgo v1.11.0
-	github.com/onsi/gomega v1.7.0
-	github.com/pkg/errors v0.8.1
-	github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf
-	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974
-	google.golang.org/grpc v1.23.0
+	github.com/blang/semver v3.5.1+incompatible
+	github.com/containernetworking/cni v0.8.1
+	github.com/containernetworking/plugins v0.9.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.3
+	github.com/pkg/errors v0.9.1
+	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	google.golang.org/grpc v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	k8s.io/api v0.18.3
-	k8s.io/apimachinery v0.18.3
-	k8s.io/client-go v0.18.3
+	k8s.io/api v0.20.10
+	k8s.io/apimachinery v0.20.10
+	k8s.io/client-go v0.20.10
 	k8s.io/klog v1.0.0
-	k8s.io/kubernetes v1.13.0
+	k8s.io/kubelet v0.0.0
+	k8s.io/kubernetes v1.20.10
 )
 
-replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+replace (
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	k8s.io/api => k8s.io/api v0.20.10
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.10
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.10
+	k8s.io/apiserver => k8s.io/apiserver v0.20.10
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.10
+	k8s.io/client-go => k8s.io/client-go v0.20.10
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.10
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.10
+	k8s.io/code-generator => k8s.io/code-generator v0.20.10
+	k8s.io/component-base => k8s.io/component-base v0.20.10
+	k8s.io/component-helpers => k8s.io/component-helpers v0.20.10
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.10
+	k8s.io/cri-api => k8s.io/cri-api v0.20.10
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.10
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.10
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.10
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.10
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.10
+	k8s.io/kubectl => k8s.io/kubectl v0.20.10
+	k8s.io/kubelet => k8s.io/kubelet v0.20.10
+	k8s.io/kubernetes => k8s.io/kubernetes v1.20.10
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.10
+	k8s.io/metrics => k8s.io/metrics v0.20.10
+	k8s.io/mount-utils => k8s.io/mount-utils v0.20.10
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.10
+)

hack/build-go.sh

@@ -21,7 +21,7 @@ if [ -z "$VERSION" ]; then
 fi
 DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
 COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
-LDFLAGS="-X main.version=${VERSION:-master} -X main.commit=${COMMIT} -X main.date=${DATE}"
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version=${VERSION:-master} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit=${COMMIT} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date=${DATE}"
 export CGO_ENABLED=0
 
 # this if... will be removed when gomodules goes default
@@ -29,7 +29,7 @@ if [ "$GO111MODULE" == "off" ]; then
 	echo "Building plugin without go module"
 	echo "Warning: this will be deprecated in near future so please use go modules!"
 
-	ORG_PATH="gopkg.in/intel"
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
 	REPO_PATH="${ORG_PATH}/multus-cni.v3"
 
 	if [ ! -h gopath/src/${REPO_PATH} ]; then
@@ -41,6 +41,8 @@ if [ "$GO111MODULE" == "off" ]; then
 	export GOBIN=${PWD}/bin
 	export GOPATH=${PWD}/gopath
 	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
+	go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/config-generation
+	go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/controller/
 else
 	# build with go modules
 	export GO111MODULE=on
@@ -51,4 +53,8 @@ else
 
 	echo "Building plugins"
 	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
+	echo "Building spec generators"
+	go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/config-generation
+	echo "Building multus controller"
+	go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/controller/
 fi

hack/test-go.sh

@@ -5,7 +5,7 @@ set -e
 if [ "$GO111MODULE" == "off" ]; then
 	echo "Warning: this will be deprecated in near future so please use go modules!"
 
-	ORG_PATH="github.com/intel"
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
 	REPO_PATH="${ORG_PATH}/multus-cni"
 
 	if [ ! -h gopath/src/${REPO_PATH} ]; then

images/Dockerfile

@@ -0,0 +1,17 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM centos:centos7
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+ADD ./images/entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm32

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "arm"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build arm container
+FROM arm32v7/centos:7
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm64

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.13.4 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -12,7 +12,9 @@ RUN  cd /usr/src/multus-cni && \
 
 # build arm64 container
 FROM arm64v8/centos:7
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 
 WORKDIR /
 ADD ./images/entrypoint.sh /

images/Dockerfile.openshift

@@ -1,5 +1,5 @@
 # This dockerfile is specific to building Multus for OpenShift
-FROM openshift/origin-release:golang-1.15 as builder
+FROM openshift/origin-release:golang-1.16 as builder
 
 ADD . /usr/src/multus-cni
 
@@ -8,6 +8,7 @@ ENV GO111MODULE=off
 RUN ./hack/build-go.sh
 
 FROM openshift/origin-base
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
 COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
 ADD ./images/entrypoint.sh /

images/Dockerfile.ppc64le

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "ppc64le"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build ppc container
+FROM ppc64le/centos:latest
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.s390x

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.13 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -12,7 +12,9 @@ RUN  cd /usr/src/multus-cni && \
 
 # build s390x container
 FROM s390x/python:3-slim
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 WORKDIR /
 ADD ./images/entrypoint.sh /
 

images/Dockerfile.thick

@@ -0,0 +1,16 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

images/README.md

@@ -5,7 +5,7 @@ This is used for distribution of Multus in a Docker image.
 Typically you'd build this from the root of your Multus clone, as such:
 
 ```
-$ docker build -t dougbtv/multus .
+$ docker build -t dougbtv/multus -f images/Dockerfile .
 ```
 
 ---
@@ -15,7 +15,7 @@ $ docker build -t dougbtv/multus .
 You may wish to deploy Multus as a daemonset, you can do so by starting with the example Daemonset shown here:
 
 ```
-$ kubectl create -f ./images/multus-daemonset.yml
+$ kubectl create -f ./deployments/multus-daemonset.yml
 ```
 
 Note: The likely best practice here is to build your own image given the Dockerfile, and then push it to your preferred registry, and change the `image` fields in the Daemonset YAML to reference that image.
@@ -41,9 +41,7 @@ in lexicographical order in cni-conf-dir).
 ./entrypoint.sh
     -h --help
     --cni-conf-dir=/host/etc/cni/net.d
-    --cni-bin-dir=/host/opt/cni/bin
     --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
-    --multus-bin-file=/usr/src/multus-cni/bin/multus
     --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
 ```
 
@@ -65,4 +63,4 @@ Example docker run command:
 $ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus
 ```
 
-Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).
+Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://docs.projectcalico.org/manifests/calico.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).

images/entrypoint.sh

@@ -18,6 +18,8 @@ MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
 MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
 MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
 MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
+MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
+MULTUS_MASTER_CNI_FILE_NAME=""
 MULTUS_NAMESPACE_ISOLATION=false
 MULTUS_GLOBAL_NAMESPACES=""
 MULTUS_LOG_TO_STDERR=true
@@ -34,22 +36,24 @@ SKIP_BINARY_COPY=false
 # Give help text for parameters.
 function usage()
 {
-    echo -e "This is an entrypoint script for Multus CNI to overlay its binary and "
-    echo -e "configuration into locations in a filesystem. The configuration & binary file "
-    echo -e "will be copied to the corresponding configuration directory. When "
-    echo -e "'--multus-conf-file=auto' is used, 00-multus.conf will be automatically "
-    echo -e "generated from the CNI configuration file of the master plugin (the first file "
-    echo -e "in lexicographical order in cni-conf-dir)."
+    echo -e "This is an entrypoint script for Multus CNI to overlay its configuration into"
+    echo -e "locations in a filesystem. The configuration file will be copied to the"
+    echo -e "corresponding configuration directory. When '--multus-conf-file=auto' is used,"
+    echo -e "00-multus.conf will be automatically generated from the CNI configuration file"
+    echo -e "of the master plugin (the first file in lexicographical order in cni-conf-dir)."
+    echo -e "When '--multus-master-cni-file-name' is used, 00-multus.conf will be"
+    echo -e "automatically generated from the specific file rather than the first file."
     echo -e ""
     echo -e "./entrypoint.sh"
     echo -e "\t-h --help"
-    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
     echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
+    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
     echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
     echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
     echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
     echo -e "\t--skip-multus-binary-copy=$SKIP_BINARY_COPY"
     echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
+    echo -e "\t--multus-master-cni-file-name=$MULTUS_MASTER_CNI_FILE_NAME (empty by default, example: 10-calico.conflist)"
     echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
     echo -e "\t--global-namespaces=$MULTUS_GLOBAL_NAMESPACES (used only with --namespace-isolation=true)"
     echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
@@ -79,10 +83,29 @@ function warn()
     log "WARN: {$1}"
 }
 
-if ! type python3 &> /dev/null; then
+if type python3 &> /dev/null; then
 	alias python=python3
 fi
 
+function checkCniVersion {
+      cniversion_python_tmpfile=$(mktemp)
+      cat << EOF > $cniversion_python_tmpfile
+import json, sys
+
+def version(v):
+    return [int(x) for x in v.split(".")]
+
+v_040 = version("0.4.0")
+v_top_level = sys.argv[2]
+with open(sys.argv[1], "r") as f:
+    v_nested = json.load(f)["cniVersion"]
+if version(v_top_level) >= v_040 and version(v_nested) < v_040:
+    msg = "Multus cni version is %s while master plugin cni version is %s"
+    print(msg % (v_top_level, v_nested))
+EOF
+      python $cniversion_python_tmpfile $1 $2
+}
+
 # Parse parameters given as arguments to this script.
 while [ "$1" != "" ]; do
     PARAM=`echo $1 | awk -F= '{print $1}'`
@@ -95,6 +118,9 @@ while [ "$1" != "" ]; do
         --cni-version)
             CNI_VERSION=$VALUE
             ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
         --cni-conf-dir)
             CNI_CONF_DIR=$VALUE
             ;;
@@ -104,12 +130,12 @@ while [ "$1" != "" ]; do
         --multus-conf-file)
             MULTUS_CONF_FILE=$VALUE
             ;;
-        --multus-bin-file)
-            MULTUS_BIN_FILE=$VALUE
-            ;;
         --multus-kubeconfig-file-host)
             MULTUS_KUBECONFIG_FILE_HOST=$VALUE
             ;;
+        --multus-master-cni-file-name)
+            MULTUS_MASTER_CNI_FILE_NAME=$VALUE
+            ;;
         --namespace-isolation)
             MULTUS_NAMESPACE_ISOLATION=$VALUE
             ;;
@@ -218,9 +244,10 @@ if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
   # to skip TLS verification for now.  We should eventually support
   # writing more complete kubeconfig files. This is only used
   # if the provided CNI network config references it.
-  touch $MULTUS_KUBECONFIG
-  chmod ${KUBECONFIG_MODE:-600} $MULTUS_KUBECONFIG
-  cat > $MULTUS_KUBECONFIG <<EOF
+  touch $MULTUS_TEMP_KUBECONFIG
+  chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
+  # Write the kubeconfig to a temp file first.
+  cat > $MULTUS_TEMP_KUBECONFIG <<EOF
 # Kubeconfig file for Multus CNI plugin.
 apiVersion: v1
 kind: Config
@@ -241,6 +268,9 @@ contexts:
 current-context: multus-context
 EOF
 
+  # Atomically move the temp kubeconfig to its permanent home.
+  mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG
+
 else
   warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
 fi
@@ -255,11 +285,15 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
   found_master=false
   tries=0
   while [ $found_master == false ]; do
-    MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
+    if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then
+        MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME"
+    else
+        MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
+    fi
     if [ "$MASTER_PLUGIN" == "" ]; then
       if [ $tries -lt 600 ]; then
         if ! (($tries % 5)); then
-          log "Attemping to find master plugin configuration, attempt $tries"
+          log "Attempting to find master plugin configuration, attempt $tries"
         fi
         let "tries+=1"
         sleep 1;
@@ -268,6 +302,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
         exit 1;
       fi
     else
+      log "Using MASTER_PLUGIN: $MASTER_PLUGIN"
 
       found_master=true
 
@@ -301,7 +336,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
           *)
               error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
               usage
-              exit 1     
+              exit 1
         esac
         LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
       fi
@@ -353,11 +388,17 @@ EOF
       NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
             python $capabilities_python_filter_tmpfile)"
       rm $capabilities_python_filter_tmpfile
-      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING" 
+      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING"
 
       MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
       MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
       log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
+      CHECK_CNI_VERSION=$(checkCniVersion $MASTER_PLUGIN_LOCATION $CNI_VERSION)
+      if [ "$CHECK_CNI_VERSION" != "" ] ; then
+        error "$CHECK_CNI_VERSION"
+        exit 1
+      fi
+
       CONF=$(cat <<-EOF
         {
           $CNI_VERSION_STRING
@@ -383,7 +424,7 @@ EOF
       mv $tmpfile $CNI_CONF_DIR/00-multus.conf
       log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
       echo $CONF
-      
+
       # If we're not performing the cleanup on exit, we can safely rename the config file.
       if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
         mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old

pkg/checkpoint/checkpoint.go

@@ -19,8 +19,8 @@ import (
 	"encoding/json"
 	"io/ioutil"
 
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 )
 

pkg/checkpoint/checkpoint_test.go

@@ -10,7 +10,7 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"

pkg/checkpoint/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package checkpoint is the package that contains the libraries that manipulates kubelet's
+// checkpoint API
+package checkpoint

pkg/config/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package config is the package that contains multus cni config related
+// utilities.
+package config

pkg/config/generator.go

@@ -0,0 +1,333 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+)
+
+const (
+	configListCapabilityKey   = "plugins"
+	singleConfigCapabilityKey = "capabilities"
+)
+
+// LogOptionFunc mutates the `LoggingOptions` object
+type LogOptionFunc func(logOptions *LogOptions)
+
+// Option mutates the `conf` object
+type Option func(conf *MultusConf)
+
+// MultusConf holds the multus configuration, and persists it to disk
+type MultusConf struct {
+	BinDir                   string          `json:"binDir,omitempty"`
+	Capabilities             map[string]bool `json:"capabilities,omitempty"`
+	CNIVersion               string          `json:"cniVersion"`
+	Delegates                []interface{}   `json:"delegates"`
+	LogFile                  string          `json:"logFile,omitempty"`
+	LogLevel                 string          `json:"logLevel,omitempty"`
+	LogToStderr              bool            `json:"logToStderr,omitempty"`
+	LogOptions               *LogOptions     `json:"logOptions,omitempty"`
+	Kubeconfig               string          `json:"kubeconfig"`
+	Name                     string          `json:"name"`
+	NamespaceIsolation       bool            `json:"namespaceIsolation,omitempty"`
+	RawNonIsolatedNamespaces string          `json:"globalNamespaces,omitempty"`
+	ReadinessIndicatorFile   string          `json:"readinessindicatorfile,omitempty"`
+	Type                     string          `json:"type"`
+}
+
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// NewMultusConfig creates a basic configuration generator. It can be mutated
+// via the `With...` methods.
+func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
+	multusConfig := &MultusConf{
+		Name:         MultusDefaultNetworkName,
+		CNIVersion:   cniVersion,
+		Type:         pluginName,
+		Capabilities: map[string]bool{},
+		Kubeconfig:   kubeconfig,
+		Delegates:    []interface{}{},
+	}
+
+	err := multusConfig.Mutate(configurationOptions...)
+	return multusConfig, err
+}
+
+// CheckVersionCompatibility checks compatibilty of the
+// top level cni version with the delegate cni version.
+// Since version 0.4.0, CHECK was introduced, which
+// causes incompatibility.
+func CheckVersionCompatibility(mc *MultusConf) error {
+	const versionFmt = "delegate cni version is %s while top level cni version is %s"
+	v040, _ := semver.Make("0.4.0")
+	multusCNIVersion, err := semver.Make(mc.CNIVersion)
+
+	if err != nil {
+		return errors.New("couldn't get top level cni version")
+	}
+
+	if multusCNIVersion.GTE(v040) {
+		for _, delegate := range mc.Delegates {
+			delegatesMap, ok := delegate.(map[string]interface{})
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			delegateVersion, ok := delegatesMap["cniVersion"].(string)
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			v, err := semver.Make(delegateVersion)
+			if err != nil {
+				return err
+			}
+			if v.LT(v040) {
+				return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Generate generates the multus configuration from whatever state is currently
+// held
+func (mc *MultusConf) Generate() (string, error) {
+	data, err := json.Marshal(mc)
+	return string(data), err
+}
+
+// Mutate updates the MultusConf attributes according to the provided
+// configuration `Option`s
+func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
+	for _, configOption := range configurationOptions {
+		configOption(mc)
+	}
+
+	return CheckVersionCompatibility(mc)
+}
+
+// WithNamespaceIsolation mutates the inner state to enable the
+// NamespaceIsolation attribute
+func WithNamespaceIsolation() Option {
+	return func(conf *MultusConf) {
+		conf.NamespaceIsolation = true
+	}
+}
+
+// WithGlobalNamespaces mutates the inner state to set the
+// RawNonIsolatedNamespaces attribute
+func WithGlobalNamespaces(globalNamespaces string) Option {
+	return func(conf *MultusConf) {
+		conf.RawNonIsolatedNamespaces = globalNamespaces
+	}
+}
+
+// WithLogToStdErr mutates the inner state to enable the
+// WithLogToStdErr attribute
+func WithLogToStdErr() Option {
+	return func(conf *MultusConf) {
+		conf.LogToStderr = true
+	}
+}
+
+// WithLogLevel mutates the inner state to set the
+// LogLevel attribute
+func WithLogLevel(logLevel string) Option {
+	return func(conf *MultusConf) {
+		conf.LogLevel = logLevel
+	}
+}
+
+// WithLogFile mutates the inner state to set the
+// logFile attribute
+func WithLogFile(logFile string) Option {
+	return func(conf *MultusConf) {
+		conf.LogFile = logFile
+	}
+}
+
+// WithLogOptions mutates the inner state to set the
+// LogOptions attribute
+func WithLogOptions(logOptions *LogOptions) Option {
+	return func(conf *MultusConf) {
+		conf.LogOptions = logOptions
+	}
+}
+
+// WithReadinessFileIndicator mutates the inner state to set the
+// ReadinessIndicatorFile attribute
+func WithReadinessFileIndicator(path string) Option {
+	return func(conf *MultusConf) {
+		conf.ReadinessIndicatorFile = path
+	}
+}
+
+// WithAdditionalBinaryFileDir mutates the inner state to set the
+// BinDir attribute
+func WithAdditionalBinaryFileDir(directoryPath string) Option {
+	return func(conf *MultusConf) {
+		conf.BinDir = directoryPath
+	}
+}
+
+// WithOverriddenName mutates the inner state to set the
+// Name attribute
+func WithOverriddenName(networkName string) Option {
+	return func(conf *MultusConf) {
+		conf.Name = networkName
+	}
+}
+
+func withCapabilities(cniData interface{}) Option {
+	var enabledCapabilities []string
+	var pluginsList []interface{}
+	cniDataMap, ok := cniData.(map[string]interface{})
+	if ok {
+		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
+			pluginsList = pluginsListEntry.([]interface{})
+		}
+	}
+
+	if len(pluginsList) > 0 {
+		for _, pluginData := range pluginsList {
+			enabledCapabilities = append(
+				enabledCapabilities,
+				extractCapabilities(pluginData)...)
+		}
+	} else {
+		enabledCapabilities = extractCapabilities(cniData)
+	}
+
+	return func(conf *MultusConf) {
+		for _, capability := range enabledCapabilities {
+			conf.Capabilities[capability] = true
+		}
+	}
+}
+
+func withDelegates(primaryCNIConfigData map[string]interface{}) Option {
+	return func(conf *MultusConf) {
+		conf.Delegates = []interface{}{primaryCNIConfigData}
+	}
+}
+
+// MutateLogOptions update the LoggingOptions of the MultusConf according
+// to the provided configuration `loggingOptions`
+func MutateLogOptions(logOption *LogOptions, logOptionFunc ...LogOptionFunc) {
+	for _, loggingOption := range logOptionFunc {
+		loggingOption(logOption)
+	}
+}
+
+// WithLogMaxSize mutates the inner state to set the
+// logMaxSize attribute
+func WithLogMaxSize(maxSize *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxSize = maxSize
+	}
+}
+
+// WithLogMaxAge mutates the inner state to set the
+// logMaxAge attribute
+func WithLogMaxAge(maxAge *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxAge = maxAge
+	}
+}
+
+// WithLogMaxBackups mutates the inner state to set the
+// logMaxBackups attribute
+func WithLogMaxBackups(maxBackups *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxBackups = maxBackups
+	}
+}
+
+// WithLogCompress mutates the inner state to set the
+// logCompress attribute
+func WithLogCompress(compress *bool) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.Compress = compress
+	}
+}
+
+func extractCapabilities(capabilitiesInterface interface{}) []string {
+	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
+	if !ok {
+		return nil
+	}
+	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	var enabledCapabilities []string
+	if len(capabilities) > 0 {
+		for capName, isCapabilityEnabled := range capabilities {
+			if isCapabilityEnabled.(bool) {
+				enabledCapabilities = append(enabledCapabilities, capName)
+			}
+		}
+	}
+	return enabledCapabilities
+}
+
+func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
+	if remainingTries == 0 {
+		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
+	}
+	var cniPluginConfigs []string
+	files, err := ioutil.ReadDir(cniConfigDirPath)
+	if err != nil {
+		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
+	}
+
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "00-multus") {
+			continue
+		}
+		fileExtension := filepath.Ext(file.Name())
+		if fileExtension == ".conf" || fileExtension == ".conflist" {
+			cniPluginConfigs = append(cniPluginConfigs, file.Name())
+		}
+	}
+
+	if len(cniPluginConfigs) == 0 {
+		time.Sleep(time.Second)
+		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
+	}
+	sort.Strings(cniPluginConfigs)
+	return cniPluginConfigs[0], nil
+}

pkg/config/generator_test.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"testing"
+)
+
+const (
+	primaryCNIName = "myCNI"
+	cniVersion     = "0.4.0"
+	kubeconfig     = "/a/b/c/kubeconfig.kubeconfig"
+)
+
+type testCase struct {
+	t                        *testing.T
+	configGenerationFunction func() (string, error)
+}
+
+var primaryCNIConfig = map[string]interface{}{
+	"cniVersion":         "1.0.0",
+	"name":               "ovn-kubernetes",
+	"type":               "ovn-k8s-cni-overlay",
+	"ipam":               "{}",
+	"dns":                "{}",
+	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+	"logLevel":           "5",
+	"logfile-maxsize":    100,
+	"logfile-maxbackups": 5,
+	"logfile-maxage":     5,
+}
+
+func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
+	multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
+	if err != nil {
+		return multusConfig, err
+	}
+	return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{})))
+}
+
+func TestBasicMultusConfig(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig)
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithNamespaceIsolation())
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithReadinessIndicator(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithReadinessFileIndicator("/a/b/u/it-lives"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogLevel("notice"),
+		WithLogToStdErr(),
+		WithLogFile("/u/y/w/log.1"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLogOptionsConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(&LogOptions{
+			MaxAge:     testutils.Int(5),
+			MaxSize:    testutils.Int(100),
+			MaxBackups: testutils.Int(5),
+			Compress:   testutils.Bool(true),
+		}))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5,\"maxSize\":100,\"maxBackups\":5,\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxAge(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxAge(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxSize(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxSize(testutils.Int(100)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxSize\":100},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogBackups(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxBackups(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxBackups\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogCompress(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogCompress(testutils.Bool(true)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithGlobalNamespace(t *testing.T) {
+	const globalNamespace = "come-along-ns"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithGlobalNamespaces(globalNamespace))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
+	const anotherCNIBinDir = "a-dir-somewhere"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithAdditionalBinaryFileDir(anotherCNIBinDir))
+	assertError(t, err, nil)
+	expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`
+{
+    "plugins": [
+        {
+            "capabilities": {
+                "portMappings": true
+            }
+        },
+        {
+            "capabilities": {
+                "tuning": false
+            }
+        }
+    ]
+}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func assertError(t *testing.T, actual error, expected error) {
+	if actual != nil && expected != nil {
+		if actual.Error() != expected.Error() {
+			t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
+		}
+	}
+
+	if actual == nil && expected != nil {
+		t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
+	} else if actual != nil && expected == nil {
+		t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
+	}
+}
+
+func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
+	return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
+}
+
+func TestVersionIncompatibility(t *testing.T) {
+	const delegateCNIVersion = "0.3.0"
+
+	primaryCNIConfigOld := primaryCNIConfig
+	tmpVer := primaryCNIConfig["cniVersion"]
+	primaryCNIConfig["cniVersion"] = delegateCNIVersion
+	_, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfigOld)
+	primaryCNIConfig["cniVersion"] = tmpVer
+
+	assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
+}
+
+func TestMultusConfigWithOverriddenName(t *testing.T) {
+	newNetworkName := "mega-net-2000"
+	multusConfig, _ := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithOverriddenName(newNetworkName))
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
+	return &testCase{
+		t:                        t,
+		configGenerationFunction: configGenerationFunc,
+	}
+}
+
+func (tc testCase) assertResult(expectedResult string) {
+	multusCNIConfig, err := tc.configGenerationFunction()
+	if err != nil {
+		tc.t.Fatalf("error generating multus configuration: %v", err)
+	}
+	if multusCNIConfig != expectedResult {
+		tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
+	}
+}
+
+func documentHelper(pluginInfo string) interface{} {
+	dp, _ := documentCNIData([]byte(pluginInfo))
+	return dp
+}
+
+func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/fsnotify/fsnotify"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+)
+
+// MultusDefaultNetworkName holds the default name of the multus network
+const (
+	multusConfigFileName     = "00-multus.conf"
+	MultusDefaultNetworkName = "multus-cni-network"
+	userRWPermission         = 0600
+)
+
+// Manager monitors the configuration of the primary CNI plugin, and
+// regenerates multus configuration whenever it gets updated.
+type Manager struct {
+	cniConfigData        map[string]interface{}
+	configWatcher        *fsnotify.Watcher
+	multusConfig         *MultusConf
+	multusConfigDir      string
+	multusConfigFilePath string
+	primaryCNIConfigPath string
+}
+
+// NewManager returns a config manager object, configured to persist the
+// configuration to `multusAutoconfigDir`. This constructor will auto-discover
+// the primary CNI for which it will delegate.
+func NewManager(config MultusConf, multusAutoconfigDir string) (*Manager, error) {
+	defaultCNIPluginName, err := primaryCNIPluginName(multusAutoconfigDir)
+	if err != nil {
+		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+		return nil, err
+	}
+	return newManager(config, multusAutoconfigDir, defaultCNIPluginName)
+}
+
+// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
+// configured to persist the configuration to `multusAutoconfigDir`. This
+// constructor will use the primary CNI plugin indicated by the user, via the
+// primaryCNIPluginName variable.
+func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir string, primaryCNIPluginName string) (*Manager, error) {
+	return newManager(config, multusAutoconfigDir, primaryCNIPluginName)
+}
+
+func newManager(config MultusConf, multusConfigDir string, defaultCNIPluginName string) (*Manager, error) {
+	watcher, err := newWatcher(multusConfigDir)
+	if err != nil {
+		return nil, err
+	}
+
+	configManager := &Manager{
+		configWatcher:        watcher,
+		multusConfig:         &config,
+		multusConfigDir:      multusConfigDir,
+		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
+		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+	}
+
+	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
+		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
+	}
+
+	return configManager, nil
+}
+
+func (m *Manager) loadPrimaryCNIConfigFromFile() error {
+	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
+	if err != nil {
+		return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
+	}
+	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
+}
+
+// OverrideNetworkName overrides the name of the multus configuration with the
+// name of the delegated primary CNI.
+func (m *Manager) OverrideNetworkName() error {
+	name, ok := m.cniConfigData["name"]
+	if !ok {
+		return fmt.Errorf("failed to access delegate CNI plugin name")
+	}
+	networkName := name.(string)
+
+	if networkName == "" {
+		return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
+	}
+	return m.multusConfig.Mutate(WithOverriddenName(networkName))
+}
+
+func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
+	cniConfigData := primaryCNIConfigData.(map[string]interface{})
+
+	m.cniConfigData = cniConfigData
+	return m.multusConfig.Mutate(
+		withDelegates(cniConfigData),
+		withCapabilities(cniConfigData))
+}
+
+// GenerateConfig generates a multus configuration from its current state
+func (m Manager) GenerateConfig() (string, error) {
+	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
+		return "", nil
+	}
+	return m.multusConfig.Generate()
+}
+
+// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
+// to by the primaryCNIPluginName attribute, and re-generates the multus
+// configuration whenever the primary CNI config is updated.
+func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
+	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
+
+	for {
+		select {
+		case event := <-m.configWatcher.Events:
+			// we're watching the DIR where the config sits, and the event
+			// does not concern the primary CNI config. Skip it.
+			if event.Name != m.primaryCNIConfigPath {
+				logging.Debugf("skipping un-related event %v", event)
+				continue
+			}
+
+			if !shouldRegenerateConfig(event) {
+				continue
+			}
+
+			updatedConfig, err := m.GenerateConfig()
+			if err != nil {
+				_ = logging.Errorf("failed to regenerate the multus configuration: %v", err)
+			}
+
+			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
+			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+			}
+			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+				_ = logging.Errorf("failed to reload the updated config: %v", err)
+			}
+
+		case err := <-m.configWatcher.Errors:
+			if err == nil {
+				continue
+			}
+			logging.Errorf("CNI monitoring error %v", err)
+
+		case <-shutDown:
+			logging.Verbosef("Stopped monitoring, closing channel ...")
+			_ = m.configWatcher.Close()
+			done <- struct{}{}
+			return nil
+		}
+	}
+}
+
+// PersistMultusConfig persists the provided configuration to the disc, with
+// Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
+func (m Manager) PersistMultusConfig(config string) error {
+	return ioutil.WriteFile(m.multusConfigFilePath, []byte(config), userRWPermission)
+}
+
+func primaryCNIPluginName(multusAutoconfigDir string) (string, error) {
+	masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
+	if err != nil {
+		return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
+	}
+	return masterCniConfigFileName, nil
+}
+
+func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
+	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
+}
+
+func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
+	}
+	defer func() {
+		// Close watcher on error
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+
+	if err = watcher.Add(cniConfigDir); err != nil {
+		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+	}
+
+	return watcher, nil
+}
+
+func shouldRegenerateConfig(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Write == fsnotify.Write ||
+		event.Op&fsnotify.Create == fsnotify.Create
+}
+
+func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
+	masterCNIConfigData, err := ioutil.ReadFile(masterCNIPluginPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
+	}
+
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall primary CNI config: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const suiteName = "Configuration Manager"
+
+func TestMultusConfigurationManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, suiteName)
+}
+
+var _ = Describe(suiteName, func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var configManager *Manager
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	BeforeEach(func() {
+		var err error
+		multusConfigDir, err = ioutil.TempDir("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+	})
+
+	BeforeEach(func() {
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), userRWPermission)).To(Succeed())
+
+		multusConf, _ := NewMultusConfig(
+			primaryCNIName,
+			cniVersion,
+			kubeconfig)
+		var err error
+		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
+		expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(config).To(Equal(expectedResult))
+	})
+
+	Context("Updates to the delegate CNI configuration", func() {
+		var (
+			doneChannel chan struct{}
+			stopChannel chan struct{}
+		)
+
+		BeforeEach(func() {
+			doneChannel = make(chan struct{})
+			stopChannel = make(chan struct{})
+			go func() {
+				Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
+			}()
+		})
+
+		AfterEach(func() {
+			go func() { stopChannel <- struct{}{} }()
+			Eventually(<-doneChannel).Should(Equal(struct{}{}))
+			close(doneChannel)
+			close(stopChannel)
+		})
+
+		It("Trigger the re-generation of the Multus CNI configuration", func() {
+			newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
+			Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), userRWPermission)).To(Succeed())
+
+			multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
+			Eventually(func() (string, error) {
+				multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
+				return string(multusCniData), err
+			}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
+		})
+	})
+
+	When("the user requests the name of the multus configuration to be overridden", func() {
+		BeforeEach(func() {
+			Expect(configManager.OverrideNetworkName()).To(Succeed())
+		})
+
+		It("Overrides the name of the multus configuration when requested", func() {
+			expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
+			config, err := configManager.GenerateConfig()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(expectedResult))
+		})
+	})
+})
+
+func multusConfigFromDelegate(delegateConfig string) string {
+	return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
+}

pkg/k8sclient/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package k8sclient is the package that contains the Kubernetes client libraries.
+package k8sclient

pkg/k8sclient/k8sclient.go

@@ -38,12 +38,12 @@ import (
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
-	"gopkg.in/intel/multus-cni.v3/pkg/kubeletclient"
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/kubeletclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 )
 
 const (
@@ -92,7 +92,7 @@ func (c *ClientInfo) Eventf(object runtime.Object, eventtype, reason, messageFmt
 	}
 }
 
-func (e *NoK8sNetworkError) Error() string { return string(e.message) }
+func (e *NoK8sNetworkError) Error() string { return e.message }
 
 // SetNetworkStatus sets network status into Pod annotation
 func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
@@ -114,11 +114,16 @@ func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []ne
 
 	podName := string(k8sArgs.K8S_POD_NAME)
 	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podUID := string(k8sArgs.K8S_POD_UID)
 	pod, err := client.GetPod(podNamespace, podName)
 	if err != nil {
 		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", podName, err)
 	}
 
+	if podUID != "" && string(pod.UID) != podUID && !IsStaticPod(pod) {
+		return logging.Errorf("SetNetworkStatus: expected pod %s/%s UID %q but got %q from Kube API", podNamespace, podName, podUID, pod.UID)
+	}
+
 	if netStatus != nil {
 		err = netutils.SetNetworkStatus(client.Client, pod, netStatus)
 		if err != nil {
@@ -259,7 +264,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
 
 		if resourceMap == nil {
-			ck, err := kubeletclient.GetResourceClient()
+			ck, err := kubeletclient.GetResourceClient("")
 			if err != nil {
 				return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a ResourceClient instance: %v", err)
 			}
@@ -361,13 +366,19 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 		}
 
 		if isGatewayConfigured == true {
-			types.CheckGatewayConfig(conf.Delegates)
+			err = types.CheckGatewayConfig(conf.Delegates)
+			if err != nil {
+				return 0, nil, err
+			}
 		}
 
-		return len(delegates), clientInfo, nil
+		return len(delegates), clientInfo, err
 	}
 
-	return 0, clientInfo, nil
+	if _, ok := err.(*NoK8sNetworkError); ok {
+		return 0, clientInfo, nil
+	}
+	return 0, clientInfo, err
 }
 
 // GetK8sClient gets client info from kubeconfig
@@ -512,7 +523,7 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 		return delegate, resourceMap, nil
 	}
 
-	// option3) search directry
+	// option3) search directory
 	fInfo, err := os.Stat(netname)
 	if err == nil {
 		if fInfo.IsDir() {
@@ -608,3 +619,16 @@ func tryLoadK8sPodDefaultNetwork(kubeClient *ClientInfo, pod *v1.Pod, conf *type
 
 	return delegate, nil
 }
+
+// ConfigSourceAnnotationKey specifies kubernetes annotation, defined in k8s.io/kubernetes/pkg/kubelet/types
+const ConfigSourceAnnotationKey = "kubernetes.io/config.source"
+
+// IsStaticPod returns true if the pod is static pod.
+func IsStaticPod(pod *v1.Pod) bool {
+	if pod.Annotations != nil {
+		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok == true {
+			return source != "api"
+		}
+	}
+	return false
+}

pkg/k8sclient/k8sclient_test.go

@@ -23,10 +23,10 @@ import (
 	"testing"
 
 	types020 "github.com/containernetworking/cni/pkg/types/020"
-	testutils "gopkg.in/intel/multus-cni.v3/pkg/testing"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
@@ -55,6 +55,9 @@ var _ = Describe("k8sclient operations", func() {
 	var tmpDir string
 	var err error
 	var genericConf string
+	var args *skel.CmdArgs
+
+	const fakePodName string = "testPod"
 
 	BeforeEach(func() {
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
@@ -69,6 +72,11 @@ var _ = Describe("k8sclient operations", func() {
 			}],
 			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
 		}`
+
+		args = &skel.CmdArgs{
+			// Values come from NewFakePod()
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePodName, "test", "testUID"),
+		}
 	})
 
 	AfterEach(func() {
@@ -77,7 +85,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using simple format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -94,10 +102,6 @@ var _ = Describe("k8sclient operations", func() {
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -130,15 +134,12 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the network does not exist", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 		net3 := `{
 	"name": "net3",
 	"type": "mynet3",
 	"cniVersion": "0.2.0"
 }`
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -160,7 +161,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using JSON format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", `[
+		fakePod := testutils.NewFakePod(fakePodName, `[
 {"name":"net1"},
 {
   "name":"net2",
@@ -172,9 +173,6 @@ var _ = Describe("k8sclient operations", func() {
   "namespace":"other-ns"
 }
 ]`, "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -218,10 +216,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the JSON format annotation is invalid", func() {
-		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "[adsfasdfasdfasf]", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -236,7 +231,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("can set the default-gateway on an additional interface", func() {
-		fakePod := testutils.NewFakePod("testpod", `[
+		fakePod := testutils.NewFakePod(fakePodName, `[
 {"name":"net1"},
 {
   "name":"net2",
@@ -247,9 +242,6 @@ var _ = Describe("k8sclient operations", func() {
   "namespace":"other-ns"
 }
 ]`, "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -294,10 +286,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using on-disk config files", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -338,10 +327,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("injects network name into minimal thick plugin CNI config", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -365,10 +351,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when on-disk config file is not valid", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
+		fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 
 		clientInfo := NewFakeClientInfo()
 		_, err := clientInfo.AddPod(fakePod)
@@ -399,7 +382,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from CRD", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -409,10 +392,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -431,7 +410,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves default networks from CRD", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -442,10 +421,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -469,7 +444,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("ignore default networks from CRD in case of kube-system namespace", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		// overwrite namespace
 		fakePod.ObjectMeta.Namespace = "kube-system"
 		conf := `{
@@ -482,10 +457,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -507,7 +478,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from file", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -518,10 +489,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf.ConfDir = tmpDir
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -546,7 +513,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from path", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -556,10 +523,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -582,7 +545,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("Error in case of CRD not found", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "")
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -592,10 +555,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -608,7 +567,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("overwrite cluster network when Pod annotation is set", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -619,10 +578,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -650,7 +605,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with bad confdir", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -661,10 +616,6 @@ var _ = Describe("k8sclient operations", func() {
 		netConf, err := types.LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -686,7 +637,7 @@ var _ = Describe("k8sclient operations", func() {
 
 	It("overwrite multus config when Pod annotation is set", func() {
 
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -701,10 +652,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -723,7 +670,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with no kubeclient and invalid kubeconfig", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -738,10 +685,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -757,7 +700,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails with no kubeclient and no kubeconfig", func() {
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -772,10 +715,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -821,7 +760,7 @@ users:
     client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMUxyZmUxaXNQaTNDYng4eHRDWG1QTlE2L0xXYVVEa1ZRNVBkdGtQK3dVYmdJUFJ6CmNtRzNEcnUya0kvamsxMGQzaUgzOVZlZDlHVjczbnZyVm5Id0pzZEF5QzVmYlRTQVBUVVNJOW90MVFGbENRUGcKQ2JLbjBRbmxUeUhXMmNnTis2Q3BWK0U4dFpWajJ2cDZoTDlmR0s2bUJCL2I4VFN3RmYxRmtnWC9hTGN1emZDaApmVFNDOWlRTk15cEFjamZadEkzWiszbHVzSVB6TVpBNTZPZXNodzlnRkJHUkw3RzZHWmZKcG9OaGVxTDlmandUClRkNURlbVZXcUxUR05ZNWhhV3hicy9VbThuM0QvdlV1QkVPYTYxQ2cvcGljUkU3Um14SmFJRWJiQlNTV1dkSDMKWDlrem5RdHJrUXloL28xZnpSV1pxeWFCY3hxR1FWN0JSSzFtSndJREFRQUJBb0lCQUJ0bjA4QzFSTU5oNjhtYgpFREV3TE1BcmEwb0JMMWNrYzN2WVFkam9XNXFVd2UwYzhQNk1YaVAweE9sTTBEbTg1a3NtdnlZSldwMFFzZXVRCnRWbldwZVNwQ015QlJPUHh2bytrRmFrdXczYk1qaktpSUN1L3EyVC96RjNzY3h4dGJIZTlVL094WGJ2YStobE0KNlpuT2ViYlpVU1A0NHNIcFVzSVNkZk1BK00ySmg1UFJibGZWaUFEY1hxNFR5RU1JaStzRkhOcFIrdmdWZzRFawp4RmFVaS83V0E2YUxWVzBUTzREdjMwbTJ0TVczWXN1bk1LTU0xOTNyUEZrU0dEdFpheWV2Z0JDeURXaFhOTEo2Clh1cTNxSUg4bFE2bzRBUjMvcDc1ZW9hOCtrVzVmT3o2UWF3WnpPYlBENkRCQlVOYVM1YklXaVV1dmx5L0JlM20KZnlxK3NRRUNnWUVBMW84R3l6ODk2bFhwdU1yVXVsb2orbGp5U0FaNkpLOCsyaFMvQnpyREx6NlpvY3FzKzg3awpVUkwzKy9LL1pja2pIMVVDeXROVVZ6Q3RKaG4zZmdLS2dpQWhsU0pNRzhqc05sbEkydFZSazNZZ1RCcUg2bXZxCit3citsTUxoUDZxbWFObUx3QXljY2lEanpMdXlRdjhVOFhKazJOdVFsQlFwbkt2eWJIRGdxSUVDZ1lFQS9kRnMKazNlYmRNNFAxV2psYXJoRTV5blpuRmdQbDg1L2Vudk4rQ1oxcStlMGxYendaUGswdWdJUWozYyt2UEpLWlh0OApLWk1HQjM0N2VLNlFIL3J1a2xRWXlLOStHeUV1YnRJQUZ2NWFrYXZxV1haR1p5ZC9QdDR1V09adXMrd3BnSG00CkxFY0lzZElsYkpFY2RJTzJyb3FaY0VNY3FEbGtXcTdwQWxqU2VxY0NnWUJYdUQ0RTFxUlByRFJVRXNrS0wxUksKUkJjNkR6dmN4N0VncEI2OXErNms0Q2tibHF0R2YvMmtqK2JISVNYVFRYcUlrczhEY1ljbjVvVEQ4UlhZZE4xLworZmNBNi9iRjNVMkZvdGRBY0xwYldZNDJ6eG9HWTN5OGluQXZEY1hkcTcxQlhML2dFc2ZiZVVycEowdm9URFdaCnlUVWwzQTZ1RzlndmI3VTdWS0xsQVFLQmdBTmNscmVOU2YzT0ROK2l1QWNsMGFQT0poZXdBdVRiMDB4bi8xNWUKQkFqMjFLbDJNaWprTkJLU25HMktBc2ExM3M1aFNFKzBwc3ZLbkRjSStOZXpseDFSQjlNQW9BYno5WTE2TW80YgphRSt0bXpqOEhBcVp0MUc1MTV0TjBnR0lDelNzYUFnT0dNdGlJU1RDOTBHRHpST2F1bFdHVGdiY1c3dm52U1pPCnp0clpBb0dBWmtIRWV5em16Z2cxR3dtTzN3bmljSHRMR1BQRFhiSW53NTdsdkIrY3lyd0FrVEs1MlFScTM0VkMKRDhnQWFwMTU2OWlWUER3YlgrNkpBQk1WQ2tNUmdxMjdHanUzN0pVY2Fib2g1YzJQeTBYNUlhUG8rek1hWHgvQwpqbjUvUW5YandjU1MrRU5hL1lXVWcxWEVjQjJYdEM0UExCdGUycitrUTVLbFNOREcxSTQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==`
 
 		kubeletconf.Write([]byte(kubeletconfDef))
-		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -836,10 +775,6 @@ users:
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
 		Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -854,7 +789,7 @@ users:
 	})
 
 	It("Errors when namespace isolation is violated", func() {
-		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -875,10 +810,6 @@ users:
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -902,7 +833,7 @@ users:
 	})
 
 	It("Properly allows a specified namespace reference when namespace isolation is enabled", func() {
-		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -924,10 +855,6 @@ users:
 	"cniVersion": "0.2.0"
 }`
 
-		args := &skel.CmdArgs{
-			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		}
-
 		clientInfo := NewFakeClientInfo()
 		_, err = clientInfo.AddPod(fakePod)
 		Expect(err).NotTo(HaveOccurred())
@@ -958,7 +885,7 @@ users:
 
 	Context("getDefaultNetDelegateCRD", func() {
 		It("fails when netConf contains bad confDir", func() {
-			fakePod := testutils.NewFakePod("testpod", "", "net1")
+			fakePod := testutils.NewFakePod(fakePodName, "", "net1")
 			conf := `{
 				"name":"node-cni-network",
 				"type":"multus",
@@ -969,10 +896,6 @@ users:
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -992,9 +915,9 @@ users:
 
 	Context("GetK8sArgs", func() {
 		It("fails when provided with bad format", func() {
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME:%s;K8S_POD_NAMESPACE:%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME:%s;K8S_POD_NAMESPACE:%s;K8S_POD_UID:%s", fakePod.Name, fakePod.Namespace, fakePod.UID),
 			}
 			// using colon instead of equals sign makes an invalid CmdArgs
 
@@ -1005,7 +928,7 @@ users:
 
 	Context("getKubernetesDelegate", func() {
 		It("failed to get a ResourceClient instance", func() {
-			fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
+			fakePod := testutils.NewFakePod(fakePodName, "net1,net2", "")
 			net1 := `{
 		"name": "net1",
 		"type": "mynet",
@@ -1021,9 +944,6 @@ users:
 		"type": "mynet3",
 		"cniVersion": "0.2.0"
 	}`
-			// args := &skel.CmdArgs{
-			// 	Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			// }
 
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
@@ -1051,10 +971,7 @@ users:
 
 	Context("parsePodNetworkObjectName", func() {
 		It("fails to get podnetwork given bad annotation values", func() {
-			fakePod := testutils.NewFakePod("testpod", "net1", "")
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
+			fakePod := testutils.NewFakePod(fakePodName, "net1", "")
 
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
@@ -1087,7 +1004,7 @@ users:
 
 	Context("setPodNetworkAnnotation", func() {
 		It("Sets pod network annotations without error", func() {
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			net1 := `{
 		"name": "net1",
@@ -1095,10 +1012,6 @@ users:
 		"cniVersion": "0.2.0"
 	}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1172,10 +1085,6 @@ users:
 		"cniVersion": "0.2.0"
 	}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err := clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1183,6 +1092,9 @@ users:
 			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
 			Expect(err).NotTo(HaveOccurred())
 
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePod.Name, fakePod.Namespace, "blahblah"),
+			}
 			k8sArgs, err := GetK8sArgs(args)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1209,7 +1121,7 @@ users:
 
 		// TODO Still figuring this next one out. deals with exponentialBackoff
 		// It("Fails to set pod network annotations without error", func() {
-		// 	fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		// 	fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 		// 	net1 := `{
 		// 	"name": "net1",
@@ -1217,10 +1129,6 @@ users:
 		// 	"cniVersion": "0.2.0"
 		// }`
 
-		// 	args := &skel.CmdArgs{
-		// 		Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-		// 	}
-
 		//	clientInfo := NewFakeClientInfo()
 		//	_, err := clientInfo.AddPod(fakePod)
 		//	Expect(err).NotTo(HaveOccurred())
@@ -1240,7 +1148,7 @@ users:
 	})
 
 	Context("SetNetworkStatus", func() {
-		It("Sets network status without error", func() {
+		It("Sets network status without error when pod UIDs match", func() {
 			result := &types020.Result{
 				CNIVersion: "0.2.0",
 				IP4: &types020.IPConfig{
@@ -1271,7 +1179,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1282,10 +1190,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1299,6 +1203,123 @@ users:
 			Expect(err).NotTo(HaveOccurred())
 		})
 
+		It("Sets pod network annotations without error when runtime does not provide a pod UID", func() {
+			result := &types020.Result{
+				CNIVersion: "0.2.0",
+				IP4: &types020.IPConfig{
+					IP: *testutils.EnsureCIDR("1.1.1.2/24"),
+				},
+			}
+
+			conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "weave-net"
+			}],
+		  "runtimeConfig": {
+			  "portMappings": [
+				{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			  ]
+			}
+		}`
+
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
+			Expect(err).NotTo(HaveOccurred())
+
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			Expect(err).NotTo(HaveOccurred())
+
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+
+			netConf, err := types.LoadNetConf([]byte(conf))
+			Expect(err).NotTo(HaveOccurred())
+
+			net1 := `{
+			"name": "net1",
+			"type": "mynet",
+			"cniVersion": "0.2.0"
+		}`
+
+			clientInfo := NewFakeClientInfo()
+			_, err = clientInfo.AddPod(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
+			Expect(err).NotTo(HaveOccurred())
+
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.Name, fakePod.Namespace),
+			}
+			k8sArgs, err := GetK8sArgs(args)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = SetNetworkStatus(clientInfo, k8sArgs, netstatus, netConf)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("Fails to set pod network annotations when pod UIDs don't match", func() {
+			result := &types020.Result{
+				CNIVersion: "0.2.0",
+				IP4: &types020.IPConfig{
+					IP: *testutils.EnsureCIDR("1.1.1.2/24"),
+				},
+			}
+
+			conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "weave-net"
+			}],
+		  "runtimeConfig": {
+			  "portMappings": [
+				{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			  ]
+			}
+		}`
+
+			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
+			Expect(err).NotTo(HaveOccurred())
+
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			Expect(err).NotTo(HaveOccurred())
+
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
+
+			netConf, err := types.LoadNetConf([]byte(conf))
+			Expect(err).NotTo(HaveOccurred())
+
+			net1 := `{
+			"name": "net1",
+			"type": "mynet",
+			"cniVersion": "0.2.0"
+		}`
+
+			clientInfo := NewFakeClientInfo()
+			_, err = clientInfo.AddPod(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = clientInfo.AddNetAttachDef(testutils.NewFakeNetAttachDef("kube-system", "net1", net1))
+			Expect(err).NotTo(HaveOccurred())
+
+			args = &skel.CmdArgs{
+				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_UID=%s", fakePod.Name, fakePod.Namespace, "foobar"),
+			}
+			k8sArgs, err := GetK8sArgs(args)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = SetNetworkStatus(clientInfo, k8sArgs, netstatus, netConf)
+			Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("expected pod %s/%s UID %q but got %q from Kube API",
+				fakePod.Namespace, fakePod.Name, string(k8sArgs.K8S_POD_UID), fakePod.UID)))
+		})
+
 		It("Sets network status with kubeclient built from kubeconfig and attempts to connect", func() {
 			kubeletconf, err := os.Create("/etc/kubernetes/kubelet.conf")
 			kubeletconfDef := `apiVersion: v1
@@ -1353,7 +1374,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1364,10 +1385,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1413,7 +1430,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1424,10 +1441,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -1472,7 +1485,7 @@ users:
 
 			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
-			fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
 			netConf, err := types.LoadNetConf([]byte(conf))
 			Expect(err).NotTo(HaveOccurred())
@@ -1483,10 +1496,6 @@ users:
 			"cniVersion": "0.2.0"
 		}`
 
-			args := &skel.CmdArgs{
-				Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
-			}
-
 			clientInfo := NewFakeClientInfo()
 			_, err = clientInfo.AddPod(fakePod)
 			Expect(err).NotTo(HaveOccurred())

pkg/kubeletclient/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package kubeletclient is the package that contains the kubelet's libraries that
+// controls podresource API in kubelet
+package kubeletclient

pkg/kubeletclient/kubeletclient.go

@@ -1,57 +1,56 @@
 package kubeletclient
 
 import (
+	"net/url"
 	"os"
-	"path/filepath"
 	"time"
 
-	"gopkg.in/intel/multus-cni.v3/pkg/checkpoint"
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	"golang.org/x/net/context"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/checkpoint"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
+	podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
 	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
-	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
 const (
 	defaultKubeletSocketFile   = "kubelet.sock"
 	defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
-)
-
-var (
-	kubeletSocket           string
-	defaultPodResourcesPath = "/var/lib/kubelet/pod-resources"
+	defaultPodResourcesPath    = "/var/lib/kubelet/pod-resources"
 )
 
 // GetResourceClient returns an instance of ResourceClient interface initialized with Pod resource information
-func GetResourceClient() (types.ResourceClient, error) {
+func GetResourceClient(kubeletSocket string) (types.ResourceClient, error) {
+	if kubeletSocket == "" {
+		kubeletSocket, _ = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
+	}
 	// If Kubelet resource API endpoint exist use that by default
 	// Or else fallback with checkpoint file
-	if hasKubeletAPIEndpoint() {
+	if hasKubeletAPIEndpoint(kubeletSocket) {
 		logging.Debugf("GetResourceClient: using Kubelet resource API endpoint")
-		return getKubeletClient()
+		return getKubeletClient(kubeletSocket)
 	}
 
 	logging.Debugf("GetResourceClient: using Kubelet device plugin checkpoint")
 	return checkpoint.GetCheckpoint()
 }
 
-func getKubeletClient() (types.ResourceClient, error) {
+func getKubeletClient(kubeletSocket string) (types.ResourceClient, error) {
 	newClient := &kubeletClient{}
 	if kubeletSocket == "" {
-		kubeletSocket = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
+		kubeletSocket, _ = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
 	}
 
-	client, conn, err := podresources.GetClient(kubeletSocket, 10*time.Second, defaultPodResourcesMaxSize)
+	client, conn, err := podresources.GetV1Client(kubeletSocket, 10*time.Second, defaultPodResourcesMaxSize)
 	if err != nil {
 		return nil, logging.Errorf("getKubeletClient: error getting grpc client: %v\n", err)
 	}
 	defer conn.Close()
 
 	if err := newClient.getPodResources(client); err != nil {
-		return nil, logging.Errorf("getKubeletClient: error ge tting pod resources from client: %v\n", err)
+		return nil, logging.Errorf("getKubeletClient: error getting pod resources from client: %v\n", err)
 	}
 
 	return newClient, nil
@@ -102,10 +101,13 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou
 	return resourceMap, nil
 }
 
-func hasKubeletAPIEndpoint() bool {
+func hasKubeletAPIEndpoint(endpoint string) bool {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return false
+	}
 	// Check for kubelet resource API socket file
-	kubeletAPISocket := filepath.Join(defaultPodResourcesPath, defaultKubeletSocketFile)
-	if _, err := os.Stat(kubeletAPISocket); err != nil {
+	if _, err := os.Stat(u.Path); err != nil {
 		logging.Debugf("hasKubeletAPIEndpoint: error looking up kubelet resource api socket file: %q", err)
 		return false
 	}

pkg/kubeletclient/kubeletclient_test.go

@@ -15,8 +15,8 @@ import (
 	k8sTypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 
-	mtypes "gopkg.in/intel/multus-cni.v3/pkg/types"
-	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
+	mtypes "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
 )
 
 var (
@@ -29,6 +29,13 @@ type fakeResourceServer struct {
 	server *grpc.Server
 }
 
+/* This is for 1.21.x or later. Uncomment it once we update vendor here!
+//TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
+func (m *fakeResourceServer) GetAllocatableResources(ctx context.Context, req *podresourcesapi.AllocatableResourcesRequest) (*podresourcesapi.AllocatableResourcesResponse, error) {
+	return &podresourcesapi.AllocatableResourcesResponse{}, nil
+}
+*/
+
 func (m *fakeResourceServer) List(ctx context.Context, req *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
 	podName := "pod-name"
 	podNamespace := "pod-namespace"
@@ -63,25 +70,28 @@ func TestKubeletclient(t *testing.T) {
 	RunSpecs(t, "Kubeletclient Suite")
 }
 
+var testKubeletSocket string
+
 func setUp() error {
 	tempSocketDir, err := ioutil.TempDir("", "kubelet-resource-client")
 	if err != nil {
 		return err
 	}
-	defaultPodResourcesPath = filepath.Join(tempSocketDir, defaultPodResourcesPath)
+	testingPodResourcesPath := filepath.Join(tempSocketDir, defaultPodResourcesPath)
 
-	if err := os.MkdirAll(defaultPodResourcesPath, os.ModeDir); err != nil {
+	if err := os.MkdirAll(testingPodResourcesPath, os.ModeDir); err != nil {
 		return err
 	}
 
-	socketDir = defaultPodResourcesPath
+	socketDir = testingPodResourcesPath
 	socketName = filepath.Join(socketDir, "kubelet.sock")
+	testKubeletSocket = socketName
 
 	fakeServer = &fakeResourceServer{server: grpc.NewServer()}
 	podresourcesapi.RegisterPodResourcesListerServer(fakeServer.server, fakeServer)
 	lis, err := util.CreateListener(socketName)
 	if err != nil {
-		return nil
+		return err
 	}
 	go fakeServer.server.Serve(lis)
 	return nil
@@ -109,14 +119,12 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 
 	Context("GetResourceClient()", func() {
 		It("should return no error", func() {
-			kubeletSocket = socketName
-			_, err := GetResourceClient()
+			_, err := GetResourceClient(testKubeletSocket)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("should fail with missing file", func() {
-			kubeletSocket = "sampleSocketString"
-			_, err := GetResourceClient()
+			_, err := GetResourceClient("sampleSocketString")
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -138,41 +146,11 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 					},
 				},
 			}
-			kubeletSocket = socketName
-			client, err := getKubeletClient()
+			client, err := getKubeletClient(testKubeletSocket)
 			Expect(err).NotTo(HaveOccurred())
 
 			outputRMap := map[string]*mtypes.ResourceInfo{
-				"resource": &mtypes.ResourceInfo{DeviceIDs: []string{"dev0", "dev1"}},
-			}
-			resourceMap, err := client.GetPodResourceMap(fakePod)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(resourceMap).ShouldNot(BeNil())
-			Expect(resourceMap).To(Equal(outputRMap))
-		})
-
-		It("should return no error with empty socket", func() {
-			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
-			fakePod := &v1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "pod-name",
-					Namespace: "pod-namespace",
-					UID:       podUID,
-				},
-				Spec: v1.PodSpec{
-					Containers: []v1.Container{
-						{
-							Name: "container-name",
-						},
-					},
-				},
-			}
-			kubeletSocket = ""
-			client, err := getKubeletClient()
-			Expect(err).NotTo(HaveOccurred())
-
-			outputRMap := map[string]*mtypes.ResourceInfo{
-				"resource": &mtypes.ResourceInfo{DeviceIDs: []string{"dev0", "dev1"}},
+				"resource": {DeviceIDs: []string{"dev0", "dev1"}},
 			}
 			resourceMap, err := client.GetPodResourceMap(fakePod)
 			Expect(err).NotTo(HaveOccurred())
@@ -181,8 +159,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 		})
 
 		It("should return an error with garbage socket value", func() {
-			kubeletSocket = "/badfilepath!?//"
-			_, err := getKubeletClient()
+			_, err := getKubeletClient("/badfilepath!?//")
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -197,8 +174,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 					UID:       podUID,
 				},
 			}
-			kubeletSocket = socketName
-			client, err := getKubeletClient()
+			client, err := getKubeletClient(testKubeletSocket)
 			Expect(err).NotTo(HaveOccurred())
 			_, err = client.GetPodResourceMap(fakePod)
 			Expect(err).To(HaveOccurred())
@@ -215,8 +191,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 					UID:       podUID,
 				},
 			}
-			kubeletSocket = socketName
-			client, err := getKubeletClient()
+			client, err := getKubeletClient(testKubeletSocket)
 			Expect(err).NotTo(HaveOccurred())
 			_, err = client.GetPodResourceMap(fakePod)
 			Expect(err).To(HaveOccurred())
@@ -234,8 +209,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 				},
 			}
 
-			kubeletSocket = socketName
-			client, err := getKubeletClient()
+			client, err := getKubeletClient(testKubeletSocket)
 			Expect(err).NotTo(HaveOccurred())
 
 			emptyRMap := map[string]*mtypes.ResourceInfo{}

pkg/logging/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package logging is the package that contains logging library.
+package logging

pkg/logging/logging.go

@@ -41,9 +41,42 @@ const (
 var loggingStderr bool
 var loggingW io.Writer
 var loggingLevel Level
+var logger *lumberjack.Logger
 
 const defaultTimestampFormat = time.RFC3339
 
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// SetLogOptions set the LoggingOptions of NetConf
+func SetLogOptions(options *LogOptions) {
+	// give some default value
+	logger.MaxSize = 100
+	logger.MaxAge = 5
+	logger.MaxBackups = 5
+	logger.Compress = true
+	if options != nil {
+		if options.MaxAge != nil {
+			logger.MaxAge = *options.MaxAge
+		}
+		if options.MaxSize != nil {
+			logger.MaxSize = *options.MaxSize
+		}
+		if options.MaxBackups != nil {
+			logger.MaxBackups = *options.MaxBackups
+		}
+		if options.Compress != nil {
+			logger.Compress = *options.Compress
+		}
+	}
+	loggingW = logger
+}
+
 func (l Level) String() string {
 	switch l {
 	case PanicLevel:
@@ -94,7 +127,7 @@ func Errorf(format string, a ...interface{}) error {
 	return fmt.Errorf(format, a...)
 }
 
-// Panicf prints logging plus stack trace. This should be used only for unrecoverble error
+// Panicf prints logging plus stack trace. This should be used only for unrecoverable error
 func Panicf(format string, a ...interface{}) {
 	printf(PanicLevel, format, a...)
 	printf(PanicLevel, "========= Stack trace output ========")
@@ -141,13 +174,8 @@ func SetLogFile(filename string) {
 		return
 	}
 
-	loggingW = &lumberjack.Logger{
-		Filename:   filename,
-		MaxSize:    100, // megabytes
-		MaxBackups: 5,
-		MaxAge:     5, // days
-		Compress:   true,
-	}
+	logger.Filename = filename
+	loggingW = logger
 
 }
 
@@ -155,4 +183,5 @@ func init() {
 	loggingStderr = true
 	loggingW = nil
 	loggingLevel = PanicLevel
+	logger = &lumberjack.Logger{}
 }

pkg/logging/logging_test.go

@@ -15,6 +15,8 @@
 package logging
 
 import (
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"gopkg.in/natefinch/lumberjack.v2"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -42,13 +44,13 @@ var _ = Describe("logging operations", func() {
 	It("Check file setter with empty", func() {
 		SetLogFile("/tmp/foobar.logging")
 		Expect(loggingW).NotTo(Equal(nil))
-		// check file existance
+		// check file existence
 	})
 
 	It("Check file setter with bad filepath", func() {
 		SetLogFile("/invalid/filepath")
 		Expect(loggingW).NotTo(Equal(nil))
-		// check file existance
+		// check file existence
 	})
 
 	It("Check loglevel setter", func() {
@@ -79,4 +81,54 @@ var _ = Describe("logging operations", func() {
 		currentLevel := loggingLevel
 		Expect(currentLevel).To(Equal(GetLoggingLevel()))
 	})
+
+	It("Check user settings logOptions for logging", func() {
+		SetLogFile("/var/log/multus.log")
+		expectLogger := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     1,
+			MaxSize:    10,
+			MaxBackups: 1,
+			Compress:   true,
+		}
+		logOptions := &LogOptions{
+			MaxAge:     testutils.Int(1),
+			MaxSize:    testutils.Int(10),
+			MaxBackups: testutils.Int(1),
+			Compress:   testutils.Bool(true),
+		}
+		SetLogOptions(logOptions)
+		Expect(expectLogger).To(Equal(logger))
+	})
+
+	It("Check user settings logOptions and missing some options", func() {
+		SetLogFile("/var/log/multus.log")
+		expectLogger := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     5,
+			MaxSize:    100,
+			MaxBackups: 1,
+			Compress:   true,
+		}
+		logOptions := &LogOptions{
+			MaxBackups: testutils.Int(1),
+			Compress:   testutils.Bool(true),
+		}
+		SetLogOptions(logOptions)
+		Expect(expectLogger).To(Equal(logger))
+	})
+
+	It("Check user don't settings logOptions for logging", func() {
+		SetLogFile("/var/log/multus.log")
+		logger1 := &lumberjack.Logger{
+			Filename:   "/var/log/multus.log",
+			MaxAge:     5,
+			MaxSize:    100,
+			MaxBackups: 5,
+			Compress:   true,
+		}
+		SetLogOptions(nil)
+		Expect(logger1).To(Equal(logger))
+	})
+
 })

pkg/multus/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package multus is the package that contains main multus function, which
+// manipulates CNI request for delegate plugins.
+package multus

pkg/multus/multus.go

@@ -17,7 +17,6 @@ package multus
 import (
 	"context"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -31,18 +30,24 @@ import (
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
-	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
-	k8s "gopkg.in/intel/multus-cni.v3/pkg/k8sclient"
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
-	"gopkg.in/intel/multus-cni.v3/pkg/netutils"
-	"gopkg.in/intel/multus-cni.v3/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/vishvananda/netlink"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	k8snet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/netutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+)
+
+const (
+	shortPollDuration = 250 * time.Millisecond
+	shortPollTimeout  = 2500 * time.Millisecond
 )
 
 var (
@@ -56,7 +61,7 @@ var (
 	pollTimeout  = 45 * time.Second
 )
 
-//PrintVersionString ...
+// PrintVersionString ...
 func PrintVersionString() string {
 	return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s",
 		version, commit, date)
@@ -161,12 +166,12 @@ func validateIfName(nsname string, ifname string) error {
 	return err
 }
 
-func confAdd(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) {
-	logging.Debugf("confAdd: %v, %s, %s", rt, string(rawNetconf), binDir)
+func confAdd(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
+	logging.Debugf("confAdd: %v, %s", rt, string(rawNetconf))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	conf, err := libcni.ConfFromBytes(rawNetconf)
 	if err != nil {
@@ -181,12 +186,12 @@ func confAdd(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invo
 	return result, nil
 }
 
-func confCheck(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invoke.Exec) error {
-	logging.Debugf("confCheck: %v, %s, %s", rt, string(rawNetconf), binDir)
+func confCheck(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("confCheck: %v, %s", rt, string(rawNetconf))
 
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	conf, err := libcni.ConfFromBytes(rawNetconf)
 	if err != nil {
@@ -195,18 +200,18 @@ func confCheck(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec in
 
 	err = cniNet.CheckNetwork(context.Background(), conf, rt)
 	if err != nil {
-		return logging.Errorf("error in getting result from DelNetwork: %v", err)
+		return logging.Errorf("error in getting result from CheckNetwork: %v", err)
 	}
 
 	return err
 }
 
-func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invoke.Exec) error {
-	logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawNetconf), binDir)
+func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("confDel: %v, %s", rt, string(rawNetconf))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	conf, err := libcni.ConfFromBytes(rawNetconf)
 	if err != nil {
@@ -221,12 +226,12 @@ func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invo
 	return err
 }
 
-func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) {
-	logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir)
+func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
+	logging.Debugf("conflistAdd: %v, %s", rt, string(rawnetconflist))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
@@ -241,12 +246,12 @@ func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, e
 	return result, nil
 }
 
-func conflistCheck(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) error {
-	logging.Debugf("conflistCheck: %v, %s, %s", rt, string(rawnetconflist), binDir)
+func conflistCheck(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("conflistCheck: %v, %s", rt, string(rawnetconflist))
 
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
@@ -261,12 +266,12 @@ func conflistCheck(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string,
 	return err
 }
 
-func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) error {
-	logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawnetconflist), binDir)
+func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("conflistDel: %v, %s", rt, string(rawnetconflist))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
-	binDirs = append([]string{binDir}, binDirs...)
-	cniNet := libcni.NewCNIConfig(binDirs, exec)
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
@@ -281,23 +286,15 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, e
 	return err
 }
 
-func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string, cniArgs string) (cnitypes.Result, error) {
-	logging.Debugf("delegateAdd: %v, %s, %v, %v, %s", exec, ifName, delegate, rt, binDir)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return nil, logging.Errorf("delegateAdd: error setting envionment variable CNI_IFNAME")
-	}
+func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
+	logging.Debugf("delegateAdd: %v, %v, %v", exec, delegate, rt)
 
-	if err := validateIfName(os.Getenv("CNI_NETNS"), ifName); err != nil {
-		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, ifName, err)
+	if err := validateIfName(rt.NetNS, rt.IfName); err != nil {
+		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, rt.IfName, err)
 	}
 
 	// Deprecated in ver 3.5.
 	if delegate.MacRequest != "" || delegate.IPRequest != nil {
-		if cniArgs != "" {
-			cniArgs = fmt.Sprintf("%s;IgnoreUnknown=true", cniArgs)
-		} else {
-			cniArgs = "IgnoreUnknown=true"
-		}
 		if delegate.MacRequest != "" {
 			// validate Mac address
 			_, err := net.ParseMAC(delegate.MacRequest)
@@ -305,8 +302,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 				return nil, logging.Errorf("delegateAdd: failed to parse mac address %q", delegate.MacRequest)
 			}
 
-			cniArgs = fmt.Sprintf("%s;MAC=%s", cniArgs, delegate.MacRequest)
-			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, ifName)
+			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"MAC", delegate.MacRequest})
 		}
 
@@ -324,8 +320,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 			}
 
 			ips := strings.Join(delegate.IPRequest, ",")
-			cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, ips)
-			logging.Debugf("delegateAdd: set IP address %q to %q", ips, ifName)
+			logging.Debugf("delegateAdd: set IP address %q to %q", ips, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"IP", ips})
 		}
 	}
@@ -333,12 +328,12 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 	var result cnitypes.Result
 	var err error
 	if delegate.ConfListPlugin {
-		result, err = conflistAdd(rt, delegate.Bytes, binDir, exec)
+		result, err = conflistAdd(rt, delegate.Bytes, multusNetconf, exec)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		result, err = confAdd(rt, delegate.Bytes, binDir, exec)
+		result, err = confAdd(rt, delegate.Bytes, multusNetconf, exec)
 		if err != nil {
 			return nil, err
 		}
@@ -379,18 +374,15 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
 		}
 	} else {
-		// for further debug https://github.com/intel/multus-cni/issues/481
+		// for further debug https://github.com/k8snetworkplumbingwg/multus-cni/issues/481
 		logging.Errorf("delegateAdd: pod nil pointer: namespace: %s, name: %s, container id: %s, pod: %v", rt.Args[1][1], rt.Args[2][1], rt.Args[3][1], pod)
 	}
 
 	return result, nil
 }
 
-func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) error {
-	logging.Debugf("delegateCheck: %v, %s, %v, %v, %s", exec, ifName, delegateConf, rt, binDir)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateCheck: error setting envionment variable CNI_IFNAME")
-	}
+func delegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("delegateCheck: %v, %v, %v", exec, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var cniConfName string
@@ -404,12 +396,12 @@ func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.Delegate
 
 	var err error
 	if delegateConf.ConfListPlugin {
-		err = conflistCheck(rt, delegateConf.Bytes, binDir, exec)
+		err = conflistCheck(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
 			return logging.Errorf("delegateCheck: error invoking ConflistCheck - %q: %v", delegateConf.ConfList.Name, err)
 		}
 	} else {
-		err = confCheck(rt, delegateConf.Bytes, binDir, exec)
+		err = confCheck(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
 			return logging.Errorf("delegateCheck: error invoking DelegateCheck - %q: %v", delegateConf.Conf.Type, err)
 		}
@@ -418,11 +410,8 @@ func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.Delegate
 	return err
 }
 
-func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) error {
-	logging.Debugf("delegateDel: %v, %v, %s, %v, %v, %s", exec, pod, ifName, delegateConf, rt, binDir)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateDel: error setting envionment variable CNI_IFNAME")
-	}
+func delegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("delegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var confName string
@@ -440,12 +429,12 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 
 	var err error
 	if delegateConf.ConfListPlugin {
-		err = conflistDel(rt, delegateConf.Bytes, binDir, exec)
+		err = conflistDel(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
 			return logging.Errorf("delegateDel: error invoking ConflistDel - %q: %v", delegateConf.ConfList.Name, err)
 		}
 	} else {
-		err = confDel(rt, delegateConf.Bytes, binDir, exec)
+		err = confDel(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
 			return logging.Errorf("delegateDel: error invoking DelegateDel - %q: %v", delegateConf.Conf.Type, err)
 		}
@@ -457,18 +446,15 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 // delPlugins deletes plugins in reverse order from lastdIdx
 // Uses netRt as base RuntimeConf (coming from NetConf) but merges it
 // with each of the delegates' configuration
-func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, binDir string) error {
-	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v, %s", exec, pod, args, k8sArgs, delegates, lastIdx, netRt, binDir)
-	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return logging.Errorf("delPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
-	}
+func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, multusNetconf *types.NetConf) error {
+	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v", exec, pod, args, k8sArgs, delegates, lastIdx, netRt)
 
 	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
 		ifName := getIfname(delegates[idx], args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
-		if err := delegateDel(exec, pod, ifName, delegates[idx], rt, binDir); err != nil {
+		if err := delegateDel(exec, pod, delegates[idx], rt, multusNetconf); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
 		if cniDeviceInfoPath != "" {
@@ -492,7 +478,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error {
 	prefix := "Multus: "
 	if k8sArgs != nil {
-		prefix += fmt.Sprintf("[%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME)
+		prefix += fmt.Sprintf("[%s/%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_UID)
 	}
 	return logging.Errorf(prefix+format, args...)
 }
@@ -500,12 +486,69 @@ func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error {
 func cmdPluginErr(k8sArgs *types.K8sArgs, confName string, format string, args ...interface{}) error {
 	msg := ""
 	if k8sArgs != nil {
-		msg += fmt.Sprintf("[%s/%s:%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, confName)
+		msg += fmt.Sprintf("[%s/%s/%s:%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_UID, confName)
 	}
 	return logging.Errorf(msg+format, args...)
 }
 
-//CmdAdd ...
+func isCriticalRequestRetriable(err error) bool {
+	logging.Debugf("isCriticalRequestRetriable: %v", err)
+	errorTypesAllowingRetry := []func(error) bool{
+		errors.IsServiceUnavailable, errors.IsInternalError, k8snet.IsConnectionReset, k8snet.IsConnectionRefused}
+	for _, f := range errorTypesAllowingRetry {
+		if f(err) {
+			return true
+		}
+	}
+	return false
+}
+
+func getPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
+	if kubeClient == nil {
+		return nil, nil
+	}
+
+	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	podUID := string(k8sArgs.K8S_POD_UID)
+
+	pod, err := kubeClient.GetPod(podNamespace, podName)
+	if err != nil {
+		// in case of a retriable error, retry 10 times with 0.25 sec interval
+		if isCriticalRequestRetriable(err) {
+			waitErr := wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
+				pod, err = kubeClient.GetPod(podNamespace, podName)
+				return pod != nil, err
+			})
+			// retry failed, then return error with retry out
+			if waitErr != nil {
+				return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
+			}
+		} else if warnOnly && errors.IsNotFound(err) {
+			// If not found, proceed to remove interface with cache
+			return nil, nil
+		} else {
+			// Other case, return error
+			return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
+		}
+	}
+
+	// In case of static pod, UID through kube api is different because of mirror pod, hence it is expected.
+	if podUID != "" && string(pod.UID) != podUID && !k8s.IsStaticPod(pod) {
+		msg := fmt.Sprintf("expected pod UID %q but got %q from Kube API", podUID, pod.UID)
+		if warnOnly {
+			// On CNI DEL we just operate on the cache when these mismatch, we don't error out.
+			// For example: stateful sets namespace/name can remain the same while podUID changes.
+			logging.Verbosef("warning: %s", msg)
+			return nil, nil
+		}
+		return nil, cmdErr(k8sArgs, msg)
+	}
+
+	return pod, nil
+}
+
+// CmdAdd ...
 func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) {
 	n, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdAdd: %v, %v, %v", args, exec, kubeClient)
@@ -529,32 +572,13 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			return err == nil, nil
 		})
 		if err != nil {
-			return nil, cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile: %v", err)
+			return nil, cmdErr(k8sArgs, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
 		}
 	}
 
-	pod := (*v1.Pod)(nil)
-	if kubeClient != nil {
-		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-		if err != nil {
-			var waitErr error
-			// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval
-			if errors.IsServiceUnavailable(err) {
-				pollDuration := 500 * time.Millisecond
-				pollTimeout := 5 * time.Second
-				waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-					return pod != nil, err
-				})
-				// retry failed, then return error with retry out
-				if waitErr != nil {
-					return nil, cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err)
-				}
-			} else {
-				// Other case, return error
-				return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
-			}
-		}
+	pod, err := getPod(kubeClient, k8sArgs, false)
+	if err != nil {
+		return nil, err
 	}
 
 	// resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation.
@@ -582,11 +606,10 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 	var result, tmpResult cnitypes.Result
 	var netStatus []nettypes.NetworkStatus
-	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
-		if cniDeviceInfoPath != "" {
+		if cniDeviceInfoPath != "" && delegate.ResourceName != "" && delegate.DeviceID != "" {
 			err = nadutils.CopyDeviceInfoForCNIFromDP(cniDeviceInfoPath, delegate.ResourceName, delegate.DeviceID)
 			// Even if the filename is set, file may not be present. Ignore error,
 			// but log and in the future may need to filter on specific errors.
@@ -595,50 +618,77 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			}
 		}
 
-		tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n.BinDir, cniArgs)
+		// We collect the delegate netName for the cachefile name as well as following errors
+		netName := delegate.Conf.Name
+		if netName == "" {
+			netName = delegate.ConfList.Name
+		}
+		tmpResult, err = delegateAdd(exec, kubeClient, pod, delegate, rt, n)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
-			netName := delegate.Conf.Name
-			if netName == "" {
-				netName = delegate.ConfList.Name
-			}
 			// Ignore errors; DEL must be idempotent anyway
-			_ = delPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n.BinDir)
+			_ = delPlugins(exec, nil, args, k8sArgs, n.Delegates, idx, n.RuntimeConfig, n)
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
 		// Remove gateway from routing table if the gateway is not used
-		deletegateway := false
+		deleteV4gateway := false
+		deleteV6gateway := false
 		adddefaultgateway := false
-		if delegate.IsFilterGateway {
-			deletegateway = true
-			logging.Debugf("Marked interface %v for gateway deletion", ifName)
+		if delegate.IsFilterV4Gateway {
+			deleteV4gateway = true
+			logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
 		} else {
 			// Otherwise, determine if this interface now gets our default route.
-			// According to 
+			// According to
 			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
 			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
 			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && delegate.GatewayRequest[0] != nil {
-				deletegateway = true
+			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+				deleteV4gateway = true
 				adddefaultgateway = true
 				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
 			}
 		}
 
-		if deletegateway {
-			tmpResult, err = netutils.DeleteDefaultGW(args, ifName, &tmpResult)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
+		if delegate.IsFilterV6Gateway {
+			deleteV6gateway = true
+			logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
+		} else {
+			// Otherwise, determine if this interface now gets our default route.
+			// According to
+			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+			// else we'll update the defaultgateway to the one specified.
+			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+				deleteV6gateway = true
+				adddefaultgateway = true
+				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
 			}
 		}
 
-		// Here we'll set the default gateway
+		// Remove gateway if `default-route` network selection is specified
+		if deleteV4gateway || deleteV6gateway {
+			err = netutils.DeleteDefaultGW(args, ifName)
+			if err != nil {
+				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
+			}
+			err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
+			if err != nil {
+				return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
+			}
+		}
+
+		// Here we'll set the default gateway which specified in `default-route` network selection
 		if adddefaultgateway {
-			tmpResult, err = netutils.SetDefaultGW(args, ifName, delegate.GatewayRequest, &tmpResult)
+			err = netutils.SetDefaultGW(args, ifName, *delegate.GatewayRequest)
 			if err != nil {
 				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
 			}
+			err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
+			if err != nil {
+				return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
+			}
 		}
 
 		// Master plugin result is always used if present
@@ -687,7 +737,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	return result, nil
 }
 
-//CmdCheck ...
+// CmdCheck ...
 func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdCheck: %v, %v, %v", args, exec, kubeClient)
@@ -704,7 +754,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 		ifName := getIfname(delegate, args.IfName, idx)
 
 		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
-		err = delegateCheck(exec, ifName, delegate, rt, in.BinDir)
+		err = delegateCheck(exec, delegate, rt, in)
 		if err != nil {
 			return err
 		}
@@ -713,7 +763,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 	return nil
 }
 
-//CmdDel ...
+// CmdDel ...
 func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
 	in, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("CmdDel: %v, %v, %v", args, exec, kubeClient)
@@ -721,18 +771,18 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return err
 	}
 
-	netnsfound := true
+	skipStatusUpdate := false
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
 		// if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
 		// so don't return an error if the device is already removed.
 		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
 		_, ok := err.(ns.NSPathNotExistErr)
+		skipStatusUpdate = true
 		if ok {
-			netnsfound = false
 			logging.Debugf("CmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
 		} else {
-			return cmdErr(nil, "failed to open netns %q: %v", netns, err)
+			logging.Debugf("CmdDel: WARNING failed to open netns %q: %v", netns, err)
 		}
 	}
 
@@ -760,31 +810,12 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return cmdErr(nil, "error getting k8s client: %v", err)
 	}
 
-	pod := (*v1.Pod)(nil)
-	if kubeClient != nil {
-		pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-		if err != nil {
-			var waitErr error
-			// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval
-			if errors.IsServiceUnavailable(err) {
-				pollDuration := 500 * time.Millisecond
-				pollTimeout := 5 * time.Second
-				waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-					pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-					return pod != nil, err
-				})
-				// retry failed, then return error with retry out
-				if waitErr != nil {
-					return cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err)
-				}
-			} else if errors.IsNotFound(err) {
-				// If not found, proceed to remove interface with cache
-				pod = nil
-			} else {
-				// Other case, return error
-				return cmdErr(k8sArgs, "error getting pod: %v", err)
-			}
-		}
+	pod, err := getPod(kubeClient, k8sArgs, true)
+	if err != nil {
+		// getPod may be failed but just do print error in its log and continue to delete
+		logging.Errorf("Multus: getPod failed: %v, but continue to delete", err)
+		// skip status update because k8s api seems to be stucked
+		skipStatusUpdate = true
 	}
 
 	// Read the cache to get delegates json for the pod
@@ -819,6 +850,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	} else {
 		defer os.Remove(path)
+		in.Delegates = []*types.DelegateNetConf{}
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
 			return cmdErr(k8sArgs, "failed to load netconf: %v", err)
 		}
@@ -837,12 +869,16 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		if v.ConfListPlugin == true && v.ConfList.CNIVersion == "" && in.CNIVersion != "" {
 			v.ConfList.CNIVersion = in.CNIVersion
 			v.Bytes, err = json.Marshal(v.ConfList)
+			if err != nil {
+				// error happen but continue to delete
+				logging.Errorf("Multus: failed to marshal delegate %q config: %v", v.Name, err)
+			}
 		}
 	}
 
 	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
 	if in.Kubeconfig != "" {
-		if netnsfound {
+		if !skipStatusUpdate {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
 				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
 				if err != nil {
@@ -851,38 +887,9 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 				}
 			}
 		} else {
-			logging.Debugf("WARNING: Unset SetNetworkStatus skipped due to netns not found.")
+			logging.Debugf("WARNING: Unset SetNetworkStatus skipped")
 		}
 	}
 
-	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in.BinDir)
-}
-
-func main() {
-	// Init command line flags to clear vendored packages' one, especially in init()
-	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-	// add version flag
-	versionOpt := false
-	flag.BoolVar(&versionOpt, "version", false, "Show application version")
-	flag.BoolVar(&versionOpt, "v", false, "Show application version")
-	flag.Parse()
-	if versionOpt == true {
-		fmt.Printf("%s\n", PrintVersionString())
-		return
-	}
-
-	skel.PluginMain(
-		func(args *skel.CmdArgs) error {
-			result, err := CmdAdd(args, nil, nil)
-			if err != nil {
-				return err
-			}
-			return result.Print()
-		},
-		func(args *skel.CmdArgs) error {
-			return CmdCheck(args, nil, nil)
-		},
-		func(args *skel.CmdArgs) error { return CmdDel(args, nil, nil) },
-		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
 }

pkg/netutils/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package netutils is the package that contains network related utilities.
+package netutils

pkg/netutils/netutils.go

@@ -16,26 +16,24 @@
 package netutils
 
 import (
-	"github.com/containernetworking/cni/pkg/skel"
-	cnitypes "github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
-	"github.com/containernetworking/plugins/pkg/ns"
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
-	"github.com/vishvananda/netlink"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"net"
-	"strings"
+	"path/filepath"
+
+	"github.com/containernetworking/cni/libcni"
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/vishvananda/netlink"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
 )
 
 // DeleteDefaultGW removes the default gateway from marked interfaces.
-func DeleteDefaultGW(args *skel.CmdArgs, ifName string, res *cnitypes.Result) (*current.Result, error) {
-	result, err := current.NewResultFromResult(*res)
-	if err != nil {
-		return nil, logging.Errorf("DeleteDefaultGW: Error creating new from current CNI result: %v", err)
-	}
-
+func DeleteDefaultGW(args *skel.CmdArgs, ifName string) error {
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
-		return nil, logging.Errorf("DeleteDefaultGW: Error getting namespace %v", err)
+		return logging.Errorf("DeleteDefaultGW: Error getting namespace %v", err)
 	}
 	defer netns.Close()
 
@@ -50,40 +48,27 @@ func DeleteDefaultGW(args *skel.CmdArgs, ifName string, res *cnitypes.Result) (*
 		}
 		return err
 	})
-	var newRoutes []*cnitypes.Route
-	for _, route := range result.Routes {
-		if mask, _ := route.Dst.Mask.Size(); mask != 0 {
-			newRoutes = append(newRoutes, route)
-		}
-	}
-	result.Routes = newRoutes
-	return result, err
+	return err
 }
 
 // SetDefaultGW adds a default gateway on a specific interface
-func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cnitypes.Result) (*current.Result, error) {
-
-	// Use the current CNI result...
-	result, err := current.NewResultFromResult(*res)
-	if err != nil {
-		return nil, logging.Errorf("SetDefaultGW: Error creating new CNI result from current: %v", err)
-	}
-
+func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP) error {
 	// This ensures we're acting within the net namespace for the pod.
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
-		return nil, logging.Errorf("SetDefaultGW: Error getting namespace %v", err)
+		return logging.Errorf("SetDefaultGW: Error getting namespace %v", err)
 	}
 	defer netns.Close()
 
-	var newResultDefaultRoutes []*cnitypes.Route
-
 	// Do this within the net namespace.
 	err = netns.Do(func(_ ns.NetNS) error {
 		var err error
 
 		// Pick up the link info as we need the index.
-		link, _ := netlink.LinkByName(ifName)
+		link, err := netlink.LinkByName(ifName)
+		if err != nil {
+			return logging.Errorf("SetDefaultGW: Error getting link %v", err)
+		}
 
 		// Cycle through all the desired gateways.
 		for _, gw := range gateways {
@@ -95,15 +80,6 @@ func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cni
 				Gw:        gw,
 			}
 
-			// Build a new element for the results route
-
-			// Set a correct CIDR depending on IP type
-			_, dstipnet, _ := net.ParseCIDR("::0/0")
-			if strings.Count(gw.String(), ":") < 2 {
-				_, dstipnet, _ = net.ParseCIDR("0.0.0.0/0")
-			}
-			newResultDefaultRoutes = append(newResultDefaultRoutes, &cnitypes.Route{Dst: *dstipnet, GW: gw})
-
 			// Perform the creation of the default route....
 			err = netlink.RouteAdd(&newDefaultRoute)
 			if err != nil {
@@ -113,7 +89,317 @@ func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP, res *cni
 		return err
 	})
 
-	result.Routes = newResultDefaultRoutes
-	return result, err
-
+	return err
+}
+
+// DeleteDefaultGWCache updates libcni cache to remove default gateway routes in result
+func DeleteDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, ipv4, ipv6 bool) error {
+	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
+
+	cache, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		return err
+	}
+	logging.Debugf("DeleteDefaultGWCache: update cache to delete GW from: %s", string(cache))
+	newCache, err := deleteDefaultGWCacheBytes(cache, ipv4, ipv6)
+	if err != nil {
+		return err
+	}
+
+	logging.Debugf("DeleteDefaultGWCache: update cache to delete GW: %s", string(newCache))
+	return ioutil.WriteFile(cacheFile, newCache, 0600)
+}
+
+func deleteDefaultGWCacheBytes(cacheFile []byte, ipv4, ipv6 bool) ([]byte, error) {
+	var cachedInfo map[string]interface{}
+	if err := json.Unmarshal(cacheFile, &cachedInfo); err != nil {
+		return nil, err
+	}
+
+	// try to get result
+	_, ok := cachedInfo["result"]
+	if !ok {
+		return nil, fmt.Errorf("cannot get result from cache")
+	}
+
+	resultJSON, ok := cachedInfo["result"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong result type: %v", cachedInfo["result"])
+	}
+	newResult, err := deleteDefaultGWResult(resultJSON, ipv4, ipv6)
+	if err != nil {
+		return nil, err
+	}
+	cachedInfo["result"] = newResult
+
+	newCache, err := json.Marshal(cachedInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode json: %v", err)
+	}
+	return newCache, nil
+}
+
+func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interface{}, error) {
+	for i, r := range routes {
+		route, ok := r.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("wrong route format: %v", r)
+		}
+		_, ok = route["dst"]
+		if ok {
+			dst, ok := route["dst"].(string)
+			if !ok {
+				return nil, fmt.Errorf("wrong dst format: %v", route["dst"])
+			}
+			if dst == dstGW {
+				routes = append(routes[:i], routes[i+1:]...)
+			}
+		}
+	}
+	return routes, nil
+}
+
+func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
+	// try to get cniVersion from result
+	_, ok := result["cniVersion"]
+	if !ok {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return deleteDefaultGWResult020(result, ipv4, ipv6)
+	}
+
+	cniVersion, ok := result["cniVersion"].(string)
+	if !ok {
+		return nil, fmt.Errorf("wrong cniVersion format: %v", result["cniVersion"])
+	}
+
+	if cniVersion == "0.1.0" || cniVersion == "0.2.0" {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return deleteDefaultGWResult020(result, ipv4, ipv6)
+	}
+
+	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+		return nil, fmt.Errorf("not supported version: %s", cniVersion)
+	}
+
+	_, ok = result["routes"]
+	if !ok {
+		// No route in result, hence we do nothing
+		return result, nil
+	}
+	routes, ok := result["routes"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong routes format: %v", result["routes"])
+	}
+
+	var err error
+	// delete IPv4 default routes
+	if ipv4 {
+		routes, err = deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ipv6 {
+		routes, err = deleteDefaultGWResultRoutes(routes, "::0/0")
+		if err != nil {
+			return nil, err
+		}
+	}
+	result["routes"] = routes
+
+	return result, nil
+}
+
+func deleteDefaultGWResult020(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
+	var err error
+	if ipv4 {
+		_, ok := result["ip4"]
+		if ok {
+			ip4, ok := result["ip4"].(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("wrong ip4 format: %v", result["ip4"])
+			}
+
+			_, ok = ip4["routes"]
+			if ok {
+				routes, ok := ip4["routes"].([]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip4 routes format: %v", ip4["routes"])
+				}
+
+				routes, err = deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+				if err != nil {
+					return nil, err
+				}
+				ip4["routes"] = routes
+			}
+		}
+	}
+
+	if ipv6 {
+		_, ok := result["ip6"]
+		if ok {
+			ip6, ok := result["ip6"].(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("wrong ip6 format: %v", result["ip6"])
+			}
+
+			_, ok = ip6["routes"]
+			if ok {
+				routes, ok := ip6["routes"].([]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip6 routes format: %v", ip6["routes"])
+				}
+
+				routes, err = deleteDefaultGWResultRoutes(routes, "::0/0")
+				if err != nil {
+					return nil, err
+				}
+				ip6["routes"] = routes
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// AddDefaultGWCache updates libcni cache to add default gateway result
+func AddDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, gw []net.IP) error {
+	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
+
+	cache, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		return err
+	}
+	logging.Debugf("AddDefaultGWCache: update cache to add GW from: %s", string(cache))
+	newCache, err := addDefaultGWCacheBytes(cache, gw)
+	if err != nil {
+		return err
+	}
+
+	logging.Debugf("AddDefaultGWCache: update cache to add GW: %s", string(newCache))
+	return ioutil.WriteFile(cacheFile, newCache, 0600)
+}
+
+func addDefaultGWCacheBytes(cacheFile []byte, gw []net.IP) ([]byte, error) {
+	var cachedInfo map[string]interface{}
+	if err := json.Unmarshal(cacheFile, &cachedInfo); err != nil {
+		return nil, err
+	}
+
+	// try to get result
+	_, ok := cachedInfo["result"]
+	if !ok {
+		return nil, fmt.Errorf("cannot get result from cache")
+	}
+
+	resultJSON, ok := cachedInfo["result"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("wrong result type: %v", cachedInfo["result"])
+	}
+	newResult, err := addDefaultGWResult(resultJSON, gw)
+	if err != nil {
+		return nil, err
+	}
+	cachedInfo["result"] = newResult
+
+	newCache, err := json.Marshal(cachedInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode json: %v", err)
+	}
+	return newCache, nil
+}
+
+func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
+	// try to get cniVersion from result
+	_, ok := result["cniVersion"]
+	if !ok {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return addDefaultGWResult020(result, gw)
+	}
+
+	cniVersion, ok := result["cniVersion"].(string)
+	if !ok {
+		return nil, fmt.Errorf("wrong cniVersion format: %v", result["cniVersion"])
+	}
+
+	if cniVersion == "0.1.0" || cniVersion == "0.2.0" {
+		// fallback to processing result for old cni version(0.1.0/0.2.0)
+		return addDefaultGWResult020(result, gw)
+	}
+
+	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+		return nil, fmt.Errorf("not supported version: %s", cniVersion)
+	}
+
+	routes := []interface{}{}
+	_, ok = result["routes"]
+	if ok {
+		routes, ok = result["routes"].([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("wrong routes format: %v", result["routes"])
+		}
+	}
+
+	for _, g := range gw {
+		dst := "0.0.0.0/0"
+		if g.To4() == nil {
+			dst = "::0/0"
+		}
+		routes = append(routes, map[string]string{
+			"dst": dst,
+			"gw":  g.String(),
+		})
+	}
+	result["routes"] = routes
+
+	return result, nil
+}
+
+func addDefaultGWResult020(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
+	for _, g := range gw {
+		if g.To4() != nil {
+			_, ok := result["ip4"]
+			if ok {
+				ip4, ok := result["ip4"].(map[string]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip4 format: %v", result["ip4"])
+				}
+				routes := []interface{}{}
+				_, ok = ip4["routes"]
+				if ok {
+					routes, ok = ip4["routes"].([]interface{})
+					if !ok {
+						return nil, fmt.Errorf("wrong ip4 routes format: %v", ip4["routes"])
+					}
+				}
+				ip4["routes"] = append(routes, map[string]string{
+					"dst": "0.0.0.0/0",
+					"gw":  g.String(),
+				})
+			}
+		} else {
+			_, ok := result["ip6"]
+			if ok {
+				ip6, ok := result["ip6"].(map[string]interface{})
+				if !ok {
+					return nil, fmt.Errorf("wrong ip6 format: %v", result["ip4"])
+				}
+				routes := []interface{}{}
+				_, ok = ip6["routes"]
+				if ok {
+					routes, ok = ip6["routes"].([]interface{})
+					if !ok {
+						return nil, fmt.Errorf("wrong ip6 routes format: %v", ip6["routes"])
+					}
+				}
+				ip6["routes"] = append(routes, map[string]string{
+					"dst": "::/0",
+					"gw":  g.String(),
+				})
+			}
+		}
+	}
+	return result, nil
 }

pkg/testing/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package testing is the package that contains unit-test libraries.
+package testing

pkg/testing/testing.go

@@ -166,3 +166,13 @@ func (r *Result) String() string {
 	}
 	return fmt.Sprintf("%sDNS:%+v", str, r.DNS)
 }
+
+// Int returns a pointer to an int
+func Int(i int) *int {
+	return &i
+}
+
+// Bool returns a pointer to a bool.
+func Bool(b bool) *bool {
+	return &b
+}

pkg/types/conf.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 
 	"github.com/containernetworking/cni/libcni"
@@ -26,7 +27,7 @@ import (
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/cni/pkg/version"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
-	"gopkg.in/intel/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
 )
 
 const (
@@ -54,18 +55,20 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 		return logging.Errorf("LoadDelegateNetConfList: a plugin delegate must have the 'type' field")
 	}
 	delegateConf.ConfListPlugin = true
+	delegateConf.Name = delegateConf.ConfList.Name
 	return nil
 }
 
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
-func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
+func LoadDelegateNetConf(bytes []byte, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
-	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
+	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), netElement, deviceID)
 
 	delegateConf := &DelegateNetConf{}
 	if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil {
 		return nil, logging.Errorf("LoadDelegateNetConf: error unmarshalling delegate config: %v", err)
 	}
+	delegateConf.Name = delegateConf.Conf.Name
 
 	// Do some minimal validation
 	if delegateConf.Conf.Type == "" {
@@ -77,9 +80,11 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf: failed to add deviceID in NetConfList bytes: %v", err)
 			}
+			delegateConf.ResourceName = resourceName
+			delegateConf.DeviceID = deviceID
 		}
-		if net != nil && net.CNIArgs != nil {
-			bytes, err = addCNIArgsInConfList(bytes, net.CNIArgs)
+		if netElement != nil && netElement.CNIArgs != nil {
+			bytes, err = addCNIArgsInConfList(bytes, netElement.CNIArgs)
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add cni-args in NetConfList bytes: %v", err)
 			}
@@ -94,45 +99,51 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 			delegateConf.ResourceName = resourceName
 			delegateConf.DeviceID = deviceID
 		}
-		if net != nil && net.CNIArgs != nil {
-			bytes, err = addCNIArgsInConfig(bytes, net.CNIArgs)
+		if netElement != nil && netElement.CNIArgs != nil {
+			bytes, err = addCNIArgsInConfig(bytes, netElement.CNIArgs)
 			if err != nil {
 				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add cni-args in NetConfList bytes: %v", err)
 			}
 		}
 	}
 
-	if net != nil {
-		if net.Name != "" {
+	if netElement != nil {
+		if netElement.Name != "" {
 			// Overwrite CNI config name with net-attach-def name
-			delegateConf.Name = fmt.Sprintf("%s/%s", net.Namespace, net.Name)
+			delegateConf.Name = fmt.Sprintf("%s/%s", netElement.Namespace, netElement.Name)
 		}
-		if net.InterfaceRequest != "" {
-			delegateConf.IfnameRequest = net.InterfaceRequest
+		if netElement.InterfaceRequest != "" {
+			delegateConf.IfnameRequest = netElement.InterfaceRequest
 		}
-		if net.MacRequest != "" {
-			delegateConf.MacRequest = net.MacRequest
+		if netElement.MacRequest != "" {
+			delegateConf.MacRequest = netElement.MacRequest
 		}
-		if net.IPRequest != nil {
-			delegateConf.IPRequest = net.IPRequest
+		if netElement.IPRequest != nil {
+			delegateConf.IPRequest = netElement.IPRequest
 		}
-		if net.BandwidthRequest != nil {
-			delegateConf.BandwidthRequest = net.BandwidthRequest
+		if netElement.BandwidthRequest != nil {
+			delegateConf.BandwidthRequest = netElement.BandwidthRequest
 		}
-		if net.PortMappingsRequest != nil {
-			delegateConf.PortMappingsRequest = net.PortMappingsRequest
+		if netElement.PortMappingsRequest != nil {
+			delegateConf.PortMappingsRequest = netElement.PortMappingsRequest
 		}
-		if net.GatewayRequest != nil {
-			delegateConf.GatewayRequest = append(delegateConf.GatewayRequest, net.GatewayRequest...)
+		if netElement.GatewayRequest != nil {
+			var list []net.IP
+			if delegateConf.GatewayRequest != nil {
+				list = append(*delegateConf.GatewayRequest, *netElement.GatewayRequest...)
+			} else {
+				list = *netElement.GatewayRequest
+			}
+			delegateConf.GatewayRequest = &list
 		}
-		if net.InfinibandGUIDRequest != "" {
-			delegateConf.InfinibandGUIDRequest = net.InfinibandGUIDRequest
+		if netElement.InfinibandGUIDRequest != "" {
+			delegateConf.InfinibandGUIDRequest = netElement.InfinibandGUIDRequest
 		}
-		if net.DeviceID != "" {
+		if netElement.DeviceID != "" {
 			if deviceID != "" {
 				logging.Debugf("Warning: Both RuntimeConfig and ResourceMap provide deviceID. Ignoring RuntimeConfig")
 			} else {
-				delegateConf.DeviceID = net.DeviceID
+				delegateConf.DeviceID = netElement.DeviceID
 			}
 		}
 	}
@@ -182,39 +193,70 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
 // CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
 // exists, merge data with the runtime config.
 func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
-	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
+	logging.Debugf("CreateCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
 	var cniDeviceInfoFile string
 	var delegateRc *RuntimeConfig
 
 	if delegate != nil {
 		delegateRc = mergeCNIRuntimeConfig(rc, delegate)
-		if delegateRc.CNIDeviceInfoFile != "" {
-			logging.Debugf("Warning: Existing value of CNIDeviceInfoFile will be overwritten %s", delegateRc.CNIDeviceInfoFile)
+		if delegateRc.DeviceID != "" {
+			if delegateRc.CNIDeviceInfoFile != "" {
+				logging.Debugf("Warning: Existing value of CNIDeviceInfoFile will be overwritten %s", delegateRc.CNIDeviceInfoFile)
+			}
+			autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName)
+			delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo)
+			cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile
+			logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile)
 		}
-		autoDeviceInfo := fmt.Sprintf("%s-%s_%s", delegate.Name, args.ContainerID, ifName)
-		delegateRc.CNIDeviceInfoFile = nadutils.GetCNIDeviceInfoPath(autoDeviceInfo)
-		cniDeviceInfoFile = delegateRc.CNIDeviceInfoFile
-		logging.Debugf("Adding auto-generated CNIDeviceInfoFile: %s", delegateRc.CNIDeviceInfoFile)
 	} else {
 		delegateRc = rc
 	}
 
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
-	// Todo
-	// ingress, egress and bandwidth capability features as same as kubelet.
 	rt := &libcni.RuntimeConf{
 		ContainerID: args.ContainerID,
 		NetNS:       args.Netns,
 		IfName:      ifName,
-		// NOTE: Verbose logging depends on this order, so please keep Args order.
+		// NOTE: Verbose logging (pod namespace/pod name)depends on this order, so please keep Args order.
 		Args: [][2]string{
 			{"IgnoreUnknown", string("true")},
 			{"K8S_POD_NAMESPACE", string(k8sArgs.K8S_POD_NAMESPACE)},
 			{"K8S_POD_NAME", string(k8sArgs.K8S_POD_NAME)},
 			{"K8S_POD_INFRA_CONTAINER_ID", string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)},
+			{"K8S_POD_UID", string(k8sArgs.K8S_POD_UID)},
 		},
 	}
 
+	// Populate rt.Args with CNI_ARGS if the rt.Args value is not set
+	cniArgs := os.Getenv("CNI_ARGS")
+	if cniArgs != "" {
+		for _, arg := range strings.Split(cniArgs, ";") {
+			// SplitN to handle = within values, like BLAH=foo=bar
+			keyval := strings.SplitN(arg, "=", 2)
+			if len(keyval) != 2 {
+				logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %s %s %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval))
+				continue
+			}
+
+			envKey := string(keyval[0])
+			envVal := string(keyval[1])
+			found := false
+			for i := range rt.Args {
+				// Update existing key if its value is empty
+				if rt.Args[i][0] == envKey && rt.Args[i][1] == "" && envVal != "" {
+					logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg)
+					rt.Args[i][1] = envVal
+					found = true
+					break
+				}
+			}
+			if !found {
+				// Add the new key if it didn't exist yet
+				rt.Args = append(rt.Args, [2]string{envKey, envVal})
+			}
+		}
+	}
+
 	if delegateRc != nil {
 		capabilityArgs := map[string]interface{}{}
 		if len(delegateRc.PortMaps) != 0 {
@@ -267,6 +309,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 
 	// Logging
 	logging.SetLogStderr(netconf.LogToStderr)
+	logging.SetLogOptions(netconf.LogOptions)
 	if netconf.LogFile != "" {
 		logging.SetLogFile(netconf.LogFile)
 	}
@@ -501,15 +544,47 @@ func addCNIArgsInConfList(inBytes []byte, cniArgs *map[string]interface{}) ([]by
 	return configBytes, nil
 }
 
-// CheckGatewayConfig check gatewayRequest and mark IsFilterGateway flag if
+// CheckGatewayConfig check gatewayRequest and mark IsFilter{V4,V6}Gateway flag if
 // gw filtering is required
-func CheckGatewayConfig(delegates []*DelegateNetConf) {
-	// Check the Gateway
-	for i, delegate := range delegates {
-		if delegate.GatewayRequest == nil {
-			delegates[i].IsFilterGateway = true
+func CheckGatewayConfig(delegates []*DelegateNetConf) error {
+
+	v4Gateways := 0
+	v6Gateways := 0
+
+	// Check the gateway
+	for _, delegate := range delegates {
+		if delegate.GatewayRequest != nil {
+			for _, gw := range *delegate.GatewayRequest {
+				if gw.To4() != nil {
+					v4Gateways++
+				} else {
+					v6Gateways++
+				}
+			}
 		}
 	}
+
+	if v4Gateways > 1 || v6Gateways > 1 {
+		return fmt.Errorf("multus does not support ECMP for default-route")
+	}
+
+	// set filter flag for each delegate
+	for i, delegate := range delegates {
+		// no GatewayRequest
+		if delegate.GatewayRequest == nil {
+			delegates[i].IsFilterV4Gateway = true
+			delegates[i].IsFilterV6Gateway = true
+		} else {
+			for _, gw := range *delegate.GatewayRequest {
+				if gw.To4() != nil {
+					delegates[i].IsFilterV6Gateway = true
+				} else {
+					delegates[i].IsFilterV4Gateway = true
+				}
+			}
+		}
+	}
+	return nil
 }
 
 // CheckSystemNamespaces checks whether given namespace is in systemNamespaces or not.

pkg/types/conf_test.go

@@ -27,7 +27,7 @@ import (
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
-	testhelpers "gopkg.in/intel/multus-cni.v3/pkg/testing"
+	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -47,7 +47,6 @@ var _ = Describe("config operations", func() {
 		var err error
 		testNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
-		os.Setenv("CNI_NETNS", testNS.Path())
 		os.Setenv("CNI_PATH", "/some/path")
 
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
@@ -114,27 +113,54 @@ var _ = Describe("config operations", func() {
 
 	It("checks if logFile and logLevel are set correctly", func() {
 		conf := `{
-	    "name": "node-cni-network",
-			"type": "multus",
-			"logLevel": "debug",
-			"logFile": "/var/log/multus.log",
-	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-	    "delegates": [{
-	        "type": "weave-net"
-	    }],
-		"runtimeConfig": {
-	      "portMappings": [
-	        {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
-	      ]
-	    }
-
-	}`
+	"name": "node-cni-network",
+	"type": "multus",
+	"logLevel": "debug",
+	"logFile": "/var/log/multus.log",
+	"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	"delegates": [{
+		"type": "weave-net"
+	}],
+	"runtimeConfig": {
+		"portMappings": [
+			{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+	}
+}`
 		netConf, err := LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 		Expect(netConf.LogLevel).To(Equal("debug"))
 		Expect(netConf.LogFile).To(Equal("/var/log/multus.log"))
 	})
 
+	It("checks if logOptions are set correctly", func() {
+		conf := `{
+	"name": "node-cni-network",
+	"type": "multus",
+	"logOptions": {
+		"maxAge": 5,
+		"maxSize": 100,
+		"maxBackups": 5,
+		"compress": true
+	},
+	"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	"delegates": [{
+		"type": "weave-net"
+	}],
+	"runtimeConfig": {
+		"portMappings": [
+			{"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+	}
+}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(*netConf.LogOptions.MaxAge).To(Equal(5))
+		Expect(*netConf.LogOptions.MaxBackups).To(Equal(5))
+		Expect(*netConf.LogOptions.MaxSize).To(Equal(100))
+		Expect(*netConf.LogOptions.Compress).To(Equal(true))
+	})
+
 	It("properly sets namespace isolation using the default namespace", func() {
 		conf := `{
 	    "name": "node-cni-network",
@@ -614,6 +640,44 @@ var _ = Describe("config operations", func() {
 		Expect(rt.CapabilityArgs["portMappings"]).To(Equal(rc.PortMaps))
 	})
 
+	It("creates a valid CNI runtime config with K8s args passed via CNI_ARGS environment variable", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "defaultnetworkwaitseconds": 3,
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    },{
+        "name": "other1",
+        "cniVersion": "0.2.0",
+        "type": "other-plugin"
+    }]
+}`),
+		}
+
+		os.Setenv("CNI_ARGS", "K8S_POD_NAME=dummy;K8S_POD_NAMESPACE=namespacedummy;K8S_POD_INFRA_CONTAINER_ID=123456789;K8S_POD_UID=aaaaa;BLAHBLAH=foo=bar")
+		k8sArgs := &K8sArgs{}
+		rt, _ := CreateCNIRuntimeConf(args, k8sArgs, "", &RuntimeConfig{}, nil)
+		fmt.Println("rt.ContainerID: ", rt.ContainerID)
+		Expect(rt.ContainerID).To(Equal("123456789"))
+		Expect(rt.NetNS).To(Equal(args.Netns))
+		Expect(rt.IfName).To(Equal(""))
+		fmt.Println("rt.ContainerID: ", rt.ContainerID)
+		Expect(rt.Args[0]).To(Equal([2]string{"IgnoreUnknown", "true"}))
+		Expect(rt.Args[1]).To(Equal([2]string{"K8S_POD_NAMESPACE", "namespacedummy"}))
+		Expect(rt.Args[2]).To(Equal([2]string{"K8S_POD_NAME", "dummy"}))
+		Expect(rt.Args[3]).To(Equal([2]string{"K8S_POD_INFRA_CONTAINER_ID", "123456789"}))
+		Expect(rt.Args[4]).To(Equal([2]string{"K8S_POD_UID", "aaaaa"}))
+		Expect(rt.Args[5]).To(Equal([2]string{"BLAHBLAH", "foo=bar"}))
+	})
+
 	It("can loadnetworkstatus", func() {
 		result := &types020.Result{
 			CNIVersion: "0.2.0",
@@ -725,6 +789,7 @@ var _ = Describe("config operations", func() {
 			"type": "multus",
 			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
 			"delegates": [{
+				"name": "weave",
 				"type": "weave-net"
 			}],
 		  "runtimeConfig": {
@@ -809,4 +874,107 @@ var _ = Describe("config operations", func() {
 		// The original RuntimeConfig must have not been overwritten
 		Expect(origRuntimeConfig).To(Equal(RuntimeConfig{}))
 	})
+
+	It("test DelegateConf Name is delivered", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave",
+				"type": "weave-net"
+			}]
+		}`
+
+		n, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(n.Delegates)).To(BeEquivalentTo(1))
+		Expect(n.Delegates[0].Name).To(Equal("weave"))
+	})
+
+	It("test DelegateConfList Name is delivered", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		n, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(n.Delegates)).To(BeEquivalentTo(1))
+		Expect(n.Delegates[0].Name).To(Equal("weave-list"))
+	})
+
+	It("test LoadDelegateNetConf keeps without GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar" }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).To(BeNil())
+	})
+
+	It("test LoadDelegateNetConf keeps empty GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar", "default-route": [] }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).NotTo(BeNil())
+		Expect(len(*netconf.GatewayRequest)).To(BeEquivalentTo(0))
+	})
+
+	It("test LoadDelegateNetConf keeps GatewayRequest", func() {
+		conf := `{
+			"name": "node-cni-network",
+			"type": "multus",
+			"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"name": "weave-list",
+				"plugins": [ {"type" :"weave"} ]
+			}]
+		}`
+
+		nsJSON := `{ "name": "foobar", "default-route": [ "10.1.1.1" ] }`
+		ns := &NetworkSelectionElement{}
+
+		err := json.Unmarshal([]byte(nsJSON), ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		netconf, err := LoadDelegateNetConf([]byte(conf), ns, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netconf.GatewayRequest).NotTo(BeNil())
+		Expect(len(*netconf.GatewayRequest)).To(BeEquivalentTo(1))
+	})
+
 })

pkg/types/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package types contains common types in the multus.
+package types

pkg/types/types.go

@@ -16,6 +16,7 @@
 package types
 
 import (
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
@@ -44,6 +45,7 @@ type NetConf struct {
 	LogFile         string                   `json:"logFile"`
 	LogLevel        string                   `json:"logLevel"`
 	LogToStderr     bool                     `json:"logToStderr,omitempty"`
+	LogOptions      *logging.LogOptions      `json:"logOptions,omitempty"`
 	RuntimeConfig   *RuntimeConfig           `json:"runtimeConfig,omitempty"`
 	// Default network readiness options
 	ReadinessIndicatorFile string `json:"readinessindicatorfile"`
@@ -97,8 +99,9 @@ type DelegateNetConf struct {
 	IPRequest             []string        `json:"ipRequest,omitempty"`
 	PortMappingsRequest   []*PortMapEntry `json:"-"`
 	BandwidthRequest      *BandwidthEntry `json:"-"`
-	GatewayRequest        []net.IP        `json:"default-route,omitempty"`
-	IsFilterGateway       bool
+	GatewayRequest        *[]net.IP       `json:"default-route,omitempty"`
+	IsFilterV4Gateway     bool
+	IsFilterV6Gateway     bool
 	// MasterPlugin is only used internal housekeeping
 	MasterPlugin bool `json:"-"`
 	// Conflist plugin is only used internal housekeeping
@@ -147,7 +150,7 @@ type NetworkSelectionElement struct {
 	// CNIArgs contains additional CNI arguments for the network interface
 	CNIArgs *map[string]interface{} `json:"cni-args"`
 	// GatewayRequest contains default route IP address for the pod
-	GatewayRequest []net.IP `json:"default-route,omitempty"`
+	GatewayRequest *[]net.IP `json:"default-route,omitempty"`
 }
 
 // K8sArgs is the valid CNI_ARGS used for Kubernetes
@@ -157,6 +160,7 @@ type K8sArgs struct {
 	K8S_POD_NAME               types.UnmarshallableString //revive:disable-line
 	K8S_POD_NAMESPACE          types.UnmarshallableString //revive:disable-line
 	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString //revive:disable-line
+	K8S_POD_UID                types.UnmarshallableString //revive:disable-line
 }
 
 // ResourceInfo is struct to hold Pod device allocation information

vendor/github.com/Microsoft/go-winio/go.mod

@@ -5,5 +5,5 @@ go 1.12
 require (
 	github.com/pkg/errors v0.8.1
 	github.com/sirupsen/logrus v1.4.1
-	golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b
+	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
 )

vendor/github.com/Microsoft/go-winio/go.sum

@@ -14,3 +14,5 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -182,13 +182,14 @@ func (s pipeAddress) String() string {
 }
 
 // tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
-func tryDialPipe(ctx context.Context, path *string) (syscall.Handle, error) {
+func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) {
 	for {
+
 		select {
 		case <-ctx.Done():
 			return syscall.Handle(0), ctx.Err()
 		default:
-			h, err := createFile(*path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
+			h, err := createFile(*path, access, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
 			if err == nil {
 				return h, nil
 			}
@@ -197,7 +198,7 @@ func tryDialPipe(ctx context.Context, path *string) (syscall.Handle, error) {
 			}
 			// Wait 10 msec and try again. This is a rather simplistic
 			// view, as we always try each 10 milliseconds.
-			time.Sleep(time.Millisecond * 10)
+			time.Sleep(10 * time.Millisecond)
 		}
 	}
 }
@@ -210,7 +211,7 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
 	if timeout != nil {
 		absTimeout = time.Now().Add(*timeout)
 	} else {
-		absTimeout = time.Now().Add(time.Second * 2)
+		absTimeout = time.Now().Add(2 * time.Second)
 	}
 	ctx, _ := context.WithDeadline(context.Background(), absTimeout)
 	conn, err := DialPipeContext(ctx, path)
@@ -223,9 +224,15 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
 // DialPipeContext attempts to connect to a named pipe by `path` until `ctx`
 // cancellation or timeout.
 func DialPipeContext(ctx context.Context, path string) (net.Conn, error) {
+	return DialPipeAccess(ctx, path, syscall.GENERIC_READ|syscall.GENERIC_WRITE)
+}
+
+// DialPipeAccess attempts to connect to a named pipe by `path` with `access` until `ctx`
+// cancellation or timeout.
+func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, error) {
 	var err error
 	var h syscall.Handle
-	h, err = tryDialPipe(ctx, &path)
+	h, err = tryDialPipe(ctx, &path, access)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/beorn7/perks/LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Blake Mizerany
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/beorn7/perks/quantile/stream.go

@@ -0,0 +1,316 @@
+// Package quantile computes approximate quantiles over an unbounded data
+// stream within low memory and CPU bounds.
+//
+// A small amount of accuracy is traded to achieve the above properties.
+//
+// Multiple streams can be merged before calling Query to generate a single set
+// of results. This is meaningful when the streams represent the same type of
+// data. See Merge and Samples.
+//
+// For more detailed information about the algorithm used, see:
+//
+// Effective Computation of Biased Quantiles over Data Streams
+//
+// http://www.cs.rutgers.edu/~muthu/bquant.pdf
+package quantile
+
+import (
+	"math"
+	"sort"
+)
+
+// Sample holds an observed value and meta information for compression. JSON
+// tags have been added for convenience.
+type Sample struct {
+	Value float64 `json:",string"`
+	Width float64 `json:",string"`
+	Delta float64 `json:",string"`
+}
+
+// Samples represents a slice of samples. It implements sort.Interface.
+type Samples []Sample
+
+func (a Samples) Len() int           { return len(a) }
+func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
+func (a Samples) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+type invariant func(s *stream, r float64) float64
+
+// NewLowBiased returns an initialized Stream for low-biased quantiles
+// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
+// error guarantees can still be given even for the lower ranks of the data
+// distribution.
+//
+// The provided epsilon is a relative error, i.e. the true quantile of a value
+// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
+//
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
+// properties.
+func NewLowBiased(epsilon float64) *Stream {
+	ƒ := func(s *stream, r float64) float64 {
+		return 2 * epsilon * r
+	}
+	return newStream(ƒ)
+}
+
+// NewHighBiased returns an initialized Stream for high-biased quantiles
+// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
+// error guarantees can still be given even for the higher ranks of the data
+// distribution.
+//
+// The provided epsilon is a relative error, i.e. the true quantile of a value
+// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
+//
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
+// properties.
+func NewHighBiased(epsilon float64) *Stream {
+	ƒ := func(s *stream, r float64) float64 {
+		return 2 * epsilon * (s.n - r)
+	}
+	return newStream(ƒ)
+}
+
+// NewTargeted returns an initialized Stream concerned with a particular set of
+// quantile values that are supplied a priori. Knowing these a priori reduces
+// space and computation time. The targets map maps the desired quantiles to
+// their absolute errors, i.e. the true quantile of a value returned by a query
+// is guaranteed to be within (Quantile±Epsilon).
+//
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
+func NewTargeted(targetMap map[float64]float64) *Stream {
+	// Convert map to slice to avoid slow iterations on a map.
+	// ƒ is called on the hot path, so converting the map to a slice
+	// beforehand results in significant CPU savings.
+	targets := targetMapToSlice(targetMap)
+
+	ƒ := func(s *stream, r float64) float64 {
+		var m = math.MaxFloat64
+		var f float64
+		for _, t := range targets {
+			if t.quantile*s.n <= r {
+				f = (2 * t.epsilon * r) / t.quantile
+			} else {
+				f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
+			}
+			if f < m {
+				m = f
+			}
+		}
+		return m
+	}
+	return newStream(ƒ)
+}
+
+type target struct {
+	quantile float64
+	epsilon  float64
+}
+
+func targetMapToSlice(targetMap map[float64]float64) []target {
+	targets := make([]target, 0, len(targetMap))
+
+	for quantile, epsilon := range targetMap {
+		t := target{
+			quantile: quantile,
+			epsilon:  epsilon,
+		}
+		targets = append(targets, t)
+	}
+
+	return targets
+}
+
+// Stream computes quantiles for a stream of float64s. It is not thread-safe by
+// design. Take care when using across multiple goroutines.
+type Stream struct {
+	*stream
+	b      Samples
+	sorted bool
+}
+
+func newStream(ƒ invariant) *Stream {
+	x := &stream{ƒ: ƒ}
+	return &Stream{x, make(Samples, 0, 500), true}
+}
+
+// Insert inserts v into the stream.
+func (s *Stream) Insert(v float64) {
+	s.insert(Sample{Value: v, Width: 1})
+}
+
+func (s *Stream) insert(sample Sample) {
+	s.b = append(s.b, sample)
+	s.sorted = false
+	if len(s.b) == cap(s.b) {
+		s.flush()
+	}
+}
+
+// Query returns the computed qth percentiles value. If s was created with
+// NewTargeted, and q is not in the set of quantiles provided a priori, Query
+// will return an unspecified result.
+func (s *Stream) Query(q float64) float64 {
+	if !s.flushed() {
+		// Fast path when there hasn't been enough data for a flush;
+		// this also yields better accuracy for small sets of data.
+		l := len(s.b)
+		if l == 0 {
+			return 0
+		}
+		i := int(math.Ceil(float64(l) * q))
+		if i > 0 {
+			i -= 1
+		}
+		s.maybeSort()
+		return s.b[i].Value
+	}
+	s.flush()
+	return s.stream.query(q)
+}
+
+// Merge merges samples into the underlying streams samples. This is handy when
+// merging multiple streams from separate threads, database shards, etc.
+//
+// ATTENTION: This method is broken and does not yield correct results. The
+// underlying algorithm is not capable of merging streams correctly.
+func (s *Stream) Merge(samples Samples) {
+	sort.Sort(samples)
+	s.stream.merge(samples)
+}
+
+// Reset reinitializes and clears the list reusing the samples buffer memory.
+func (s *Stream) Reset() {
+	s.stream.reset()
+	s.b = s.b[:0]
+}
+
+// Samples returns stream samples held by s.
+func (s *Stream) Samples() Samples {
+	if !s.flushed() {
+		return s.b
+	}
+	s.flush()
+	return s.stream.samples()
+}
+
+// Count returns the total number of samples observed in the stream
+// since initialization.
+func (s *Stream) Count() int {
+	return len(s.b) + s.stream.count()
+}
+
+func (s *Stream) flush() {
+	s.maybeSort()
+	s.stream.merge(s.b)
+	s.b = s.b[:0]
+}
+
+func (s *Stream) maybeSort() {
+	if !s.sorted {
+		s.sorted = true
+		sort.Sort(s.b)
+	}
+}
+
+func (s *Stream) flushed() bool {
+	return len(s.stream.l) > 0
+}
+
+type stream struct {
+	n float64
+	l []Sample
+	ƒ invariant
+}
+
+func (s *stream) reset() {
+	s.l = s.l[:0]
+	s.n = 0
+}
+
+func (s *stream) insert(v float64) {
+	s.merge(Samples{{v, 1, 0}})
+}
+
+func (s *stream) merge(samples Samples) {
+	// TODO(beorn7): This tries to merge not only individual samples, but
+	// whole summaries. The paper doesn't mention merging summaries at
+	// all. Unittests show that the merging is inaccurate. Find out how to
+	// do merges properly.
+	var r float64
+	i := 0
+	for _, sample := range samples {
+		for ; i < len(s.l); i++ {
+			c := s.l[i]
+			if c.Value > sample.Value {
+				// Insert at position i.
+				s.l = append(s.l, Sample{})
+				copy(s.l[i+1:], s.l[i:])
+				s.l[i] = Sample{
+					sample.Value,
+					sample.Width,
+					math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
+					// TODO(beorn7): How to calculate delta correctly?
+				}
+				i++
+				goto inserted
+			}
+			r += c.Width
+		}
+		s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
+		i++
+	inserted:
+		s.n += sample.Width
+		r += sample.Width
+	}
+	s.compress()
+}
+
+func (s *stream) count() int {
+	return int(s.n)
+}
+
+func (s *stream) query(q float64) float64 {
+	t := math.Ceil(q * s.n)
+	t += math.Ceil(s.ƒ(s, t) / 2)
+	p := s.l[0]
+	var r float64
+	for _, c := range s.l[1:] {
+		r += p.Width
+		if r+c.Width+c.Delta > t {
+			return p.Value
+		}
+		p = c
+	}
+	return p.Value
+}
+
+func (s *stream) compress() {
+	if len(s.l) < 2 {
+		return
+	}
+	x := s.l[len(s.l)-1]
+	xi := len(s.l) - 1
+	r := s.n - 1 - x.Width
+
+	for i := len(s.l) - 2; i >= 0; i-- {
+		c := s.l[i]
+		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
+			x.Width += c.Width
+			s.l[xi] = x
+			// Remove element at i.
+			copy(s.l[i:], s.l[i+1:])
+			s.l = s.l[:len(s.l)-1]
+			xi -= 1
+		} else {
+			x = c
+			xi = i
+		}
+		r -= c.Width
+	}
+}
+
+func (s *stream) samples() Samples {
+	samples := make(Samples, len(s.l))
+	copy(samples, s.l)
+	return samples
+}

vendor/github.com/blang/semver/.travis.yml

@@ -0,0 +1,21 @@
+language: go
+matrix:
+  include:
+  - go: 1.4.3
+  - go: 1.5.4
+  - go: 1.6.3
+  - go: 1.7
+  - go: tip
+  allow_failures:
+  - go: tip
+install:
+- go get golang.org/x/tools/cmd/cover
+- go get github.com/mattn/goveralls
+script:
+- echo "Test and track coverage" ; $HOME/gopath/bin/goveralls -package "." -service=travis-ci
+  -repotoken $COVERALLS_TOKEN
+- echo "Build examples" ; cd examples && go build
+- echo "Check if gofmt'd" ; diff -u <(echo -n) <(gofmt -d -s .)
+env:
+  global:
+    secure: HroGEAUQpVq9zX1b1VIkraLiywhGbzvNnTZq2TMxgK7JHP8xqNplAeF1izrR2i4QLL9nsY+9WtYss4QuPvEtZcVHUobw6XnL6radF7jS1LgfYZ9Y7oF+zogZ2I5QUMRLGA7rcxQ05s7mKq3XZQfeqaNts4bms/eZRefWuaFZbkw=

vendor/github.com/blang/semver/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2014 Benedikt Lang <github at benediktlang.de>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

vendor/github.com/blang/semver/README.md

@@ -0,0 +1,194 @@
+semver for golang [![Build Status](https://travis-ci.org/blang/semver.svg?branch=master)](https://travis-ci.org/blang/semver) [![GoDoc](https://godoc.org/github.com/blang/semver?status.png)](https://godoc.org/github.com/blang/semver) [![Coverage Status](https://img.shields.io/coveralls/blang/semver.svg)](https://coveralls.io/r/blang/semver?branch=master)
+======
+
+semver is a [Semantic Versioning](http://semver.org/) library written in golang. It fully covers spec version `2.0.0`.
+
+Usage
+-----
+```bash
+$ go get github.com/blang/semver
+```
+Note: Always vendor your dependencies or fix on a specific version tag.
+
+```go
+import github.com/blang/semver
+v1, err := semver.Make("1.0.0-beta")
+v2, err := semver.Make("2.0.0-beta")
+v1.Compare(v2)
+```
+
+Also check the [GoDocs](http://godoc.org/github.com/blang/semver).
+
+Why should I use this lib?
+-----
+
+- Fully spec compatible
+- No reflection
+- No regex
+- Fully tested (Coverage >99%)
+- Readable parsing/validation errors
+- Fast (See [Benchmarks](#benchmarks))
+- Only Stdlib
+- Uses values instead of pointers
+- Many features, see below
+
+
+Features
+-----
+
+- Parsing and validation at all levels
+- Comparator-like comparisons
+- Compare Helper Methods
+- InPlace manipulation
+- Ranges `>=1.0.0 <2.0.0 || >=3.0.0 !3.0.1-beta.1`
+- Wildcards `>=1.x`, `<=2.5.x`
+- Sortable (implements sort.Interface)
+- database/sql compatible (sql.Scanner/Valuer)
+- encoding/json compatible (json.Marshaler/Unmarshaler)
+
+Ranges
+------
+
+A `Range` is a set of conditions which specify which versions satisfy the range.
+
+A condition is composed of an operator and a version. The supported operators are:
+
+- `<1.0.0` Less than `1.0.0`
+- `<=1.0.0` Less than or equal to `1.0.0`
+- `>1.0.0` Greater than `1.0.0`
+- `>=1.0.0` Greater than or equal to `1.0.0`
+- `1.0.0`, `=1.0.0`, `==1.0.0` Equal to `1.0.0`
+- `!1.0.0`, `!=1.0.0` Not equal to `1.0.0`. Excludes version `1.0.0`.
+
+Note that spaces between the operator and the version will be gracefully tolerated.
+
+A `Range` can link multiple `Ranges` separated by space:
+
+Ranges can be linked by logical AND:
+
+  - `>1.0.0 <2.0.0` would match between both ranges, so `1.1.1` and `1.8.7` but not `1.0.0` or `2.0.0`
+  - `>1.0.0 <3.0.0 !2.0.3-beta.2` would match every version between `1.0.0` and `3.0.0` except `2.0.3-beta.2`
+
+Ranges can also be linked by logical OR:
+
+  - `<2.0.0 || >=3.0.0` would match `1.x.x` and `3.x.x` but not `2.x.x`
+
+AND has a higher precedence than OR. It's not possible to use brackets.
+
+Ranges can be combined by both AND and OR
+
+  - `>1.0.0 <2.0.0 || >3.0.0 !4.2.1` would match `1.2.3`, `1.9.9`, `3.1.1`, but not `4.2.1`, `2.1.1`
+
+Range usage:
+
+```
+v, err := semver.Parse("1.2.3")
+range, err := semver.ParseRange(">1.0.0 <2.0.0 || >=3.0.0")
+if range(v) {
+    //valid
+}
+
+```
+
+Example
+-----
+
+Have a look at full examples in [examples/main.go](examples/main.go)
+
+```go
+import github.com/blang/semver
+
+v, err := semver.Make("0.0.1-alpha.preview+123.github")
+fmt.Printf("Major: %d\n", v.Major)
+fmt.Printf("Minor: %d\n", v.Minor)
+fmt.Printf("Patch: %d\n", v.Patch)
+fmt.Printf("Pre: %s\n", v.Pre)
+fmt.Printf("Build: %s\n", v.Build)
+
+// Prerelease versions array
+if len(v.Pre) > 0 {
+    fmt.Println("Prerelease versions:")
+    for i, pre := range v.Pre {
+        fmt.Printf("%d: %q\n", i, pre)
+    }
+}
+
+// Build meta data array
+if len(v.Build) > 0 {
+    fmt.Println("Build meta data:")
+    for i, build := range v.Build {
+        fmt.Printf("%d: %q\n", i, build)
+    }
+}
+
+v001, err := semver.Make("0.0.1")
+// Compare using helpers: v.GT(v2), v.LT, v.GTE, v.LTE
+v001.GT(v) == true
+v.LT(v001) == true
+v.GTE(v) == true
+v.LTE(v) == true
+
+// Or use v.Compare(v2) for comparisons (-1, 0, 1):
+v001.Compare(v) == 1
+v.Compare(v001) == -1
+v.Compare(v) == 0
+
+// Manipulate Version in place:
+v.Pre[0], err = semver.NewPRVersion("beta")
+if err != nil {
+    fmt.Printf("Error parsing pre release version: %q", err)
+}
+
+fmt.Println("\nValidate versions:")
+v.Build[0] = "?"
+
+err = v.Validate()
+if err != nil {
+    fmt.Printf("Validation failed: %s\n", err)
+}
+```
+
+
+Benchmarks
+-----
+
+    BenchmarkParseSimple-4           5000000    390    ns/op    48 B/op   1 allocs/op
+    BenchmarkParseComplex-4          1000000   1813    ns/op   256 B/op   7 allocs/op
+    BenchmarkParseAverage-4          1000000   1171    ns/op   163 B/op   4 allocs/op
+    BenchmarkStringSimple-4         20000000    119    ns/op    16 B/op   1 allocs/op
+    BenchmarkStringLarger-4         10000000    206    ns/op    32 B/op   2 allocs/op
+    BenchmarkStringComplex-4         5000000    324    ns/op    80 B/op   3 allocs/op
+    BenchmarkStringAverage-4         5000000    273    ns/op    53 B/op   2 allocs/op
+    BenchmarkValidateSimple-4      200000000      9.33 ns/op     0 B/op   0 allocs/op
+    BenchmarkValidateComplex-4       3000000    469    ns/op     0 B/op   0 allocs/op
+    BenchmarkValidateAverage-4       5000000    256    ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareSimple-4       100000000     11.8  ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareComplex-4       50000000     30.8  ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareAverage-4       30000000     41.5  ns/op     0 B/op   0 allocs/op
+    BenchmarkSort-4                  3000000    419    ns/op   256 B/op   2 allocs/op
+    BenchmarkRangeParseSimple-4      2000000    850    ns/op   192 B/op   5 allocs/op
+    BenchmarkRangeParseAverage-4     1000000   1677    ns/op   400 B/op  10 allocs/op
+    BenchmarkRangeParseComplex-4      300000   5214    ns/op  1440 B/op  30 allocs/op
+    BenchmarkRangeMatchSimple-4     50000000     25.6  ns/op     0 B/op   0 allocs/op
+    BenchmarkRangeMatchAverage-4    30000000     56.4  ns/op     0 B/op   0 allocs/op
+    BenchmarkRangeMatchComplex-4    10000000    153    ns/op     0 B/op   0 allocs/op
+
+See benchmark cases at [semver_test.go](semver_test.go)
+
+
+Motivation
+-----
+
+I simply couldn't find any lib supporting the full spec. Others were just wrong or used reflection and regex which i don't like.
+
+
+Contribution
+-----
+
+Feel free to make a pull request. For bigger changes create a issue first to discuss about it.
+
+
+License
+-----
+
+See [LICENSE](LICENSE) file.

vendor/github.com/blang/semver/json.go

@@ -0,0 +1,23 @@
+package semver
+
+import (
+	"encoding/json"
+)
+
+// MarshalJSON implements the encoding/json.Marshaler interface.
+func (v Version) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.String())
+}
+
+// UnmarshalJSON implements the encoding/json.Unmarshaler interface.
+func (v *Version) UnmarshalJSON(data []byte) (err error) {
+	var versionString string
+
+	if err = json.Unmarshal(data, &versionString); err != nil {
+		return
+	}
+
+	*v, err = Parse(versionString)
+
+	return
+}

vendor/github.com/blang/semver/package.json

@@ -0,0 +1,17 @@
+{
+  "author": "blang",
+  "bugs": {
+    "URL": "https://github.com/blang/semver/issues",
+    "url": "https://github.com/blang/semver/issues"
+  },
+  "gx": {
+    "dvcsimport": "github.com/blang/semver"
+  },
+  "gxVersion": "0.10.0",
+  "language": "go",
+  "license": "MIT",
+  "name": "semver",
+  "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
+  "version": "3.5.1"
+}
+

vendor/github.com/blang/semver/range.go

@@ -0,0 +1,416 @@
+package semver
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+type wildcardType int
+
+const (
+	noneWildcard  wildcardType = iota
+	majorWildcard wildcardType = 1
+	minorWildcard wildcardType = 2
+	patchWildcard wildcardType = 3
+)
+
+func wildcardTypefromInt(i int) wildcardType {
+	switch i {
+	case 1:
+		return majorWildcard
+	case 2:
+		return minorWildcard
+	case 3:
+		return patchWildcard
+	default:
+		return noneWildcard
+	}
+}
+
+type comparator func(Version, Version) bool
+
+var (
+	compEQ comparator = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == 0
+	}
+	compNE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) != 0
+	}
+	compGT = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == 1
+	}
+	compGE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) >= 0
+	}
+	compLT = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == -1
+	}
+	compLE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) <= 0
+	}
+)
+
+type versionRange struct {
+	v Version
+	c comparator
+}
+
+// rangeFunc creates a Range from the given versionRange.
+func (vr *versionRange) rangeFunc() Range {
+	return Range(func(v Version) bool {
+		return vr.c(v, vr.v)
+	})
+}
+
+// Range represents a range of versions.
+// A Range can be used to check if a Version satisfies it:
+//
+//     range, err := semver.ParseRange(">1.0.0 <2.0.0")
+//     range(semver.MustParse("1.1.1") // returns true
+type Range func(Version) bool
+
+// OR combines the existing Range with another Range using logical OR.
+func (rf Range) OR(f Range) Range {
+	return Range(func(v Version) bool {
+		return rf(v) || f(v)
+	})
+}
+
+// AND combines the existing Range with another Range using logical AND.
+func (rf Range) AND(f Range) Range {
+	return Range(func(v Version) bool {
+		return rf(v) && f(v)
+	})
+}
+
+// ParseRange parses a range and returns a Range.
+// If the range could not be parsed an error is returned.
+//
+// Valid ranges are:
+//   - "<1.0.0"
+//   - "<=1.0.0"
+//   - ">1.0.0"
+//   - ">=1.0.0"
+//   - "1.0.0", "=1.0.0", "==1.0.0"
+//   - "!1.0.0", "!=1.0.0"
+//
+// A Range can consist of multiple ranges separated by space:
+// Ranges can be linked by logical AND:
+//   - ">1.0.0 <2.0.0" would match between both ranges, so "1.1.1" and "1.8.7" but not "1.0.0" or "2.0.0"
+//   - ">1.0.0 <3.0.0 !2.0.3-beta.2" would match every version between 1.0.0 and 3.0.0 except 2.0.3-beta.2
+//
+// Ranges can also be linked by logical OR:
+//   - "<2.0.0 || >=3.0.0" would match "1.x.x" and "3.x.x" but not "2.x.x"
+//
+// AND has a higher precedence than OR. It's not possible to use brackets.
+//
+// Ranges can be combined by both AND and OR
+//
+//  - `>1.0.0 <2.0.0 || >3.0.0 !4.2.1` would match `1.2.3`, `1.9.9`, `3.1.1`, but not `4.2.1`, `2.1.1`
+func ParseRange(s string) (Range, error) {
+	parts := splitAndTrim(s)
+	orParts, err := splitORParts(parts)
+	if err != nil {
+		return nil, err
+	}
+	expandedParts, err := expandWildcardVersion(orParts)
+	if err != nil {
+		return nil, err
+	}
+	var orFn Range
+	for _, p := range expandedParts {
+		var andFn Range
+		for _, ap := range p {
+			opStr, vStr, err := splitComparatorVersion(ap)
+			if err != nil {
+				return nil, err
+			}
+			vr, err := buildVersionRange(opStr, vStr)
+			if err != nil {
+				return nil, fmt.Errorf("Could not parse Range %q: %s", ap, err)
+			}
+			rf := vr.rangeFunc()
+
+			// Set function
+			if andFn == nil {
+				andFn = rf
+			} else { // Combine with existing function
+				andFn = andFn.AND(rf)
+			}
+		}
+		if orFn == nil {
+			orFn = andFn
+		} else {
+			orFn = orFn.OR(andFn)
+		}
+
+	}
+	return orFn, nil
+}
+
+// splitORParts splits the already cleaned parts by '||'.
+// Checks for invalid positions of the operator and returns an
+// error if found.
+func splitORParts(parts []string) ([][]string, error) {
+	var ORparts [][]string
+	last := 0
+	for i, p := range parts {
+		if p == "||" {
+			if i == 0 {
+				return nil, fmt.Errorf("First element in range is '||'")
+			}
+			ORparts = append(ORparts, parts[last:i])
+			last = i + 1
+		}
+	}
+	if last == len(parts) {
+		return nil, fmt.Errorf("Last element in range is '||'")
+	}
+	ORparts = append(ORparts, parts[last:])
+	return ORparts, nil
+}
+
+// buildVersionRange takes a slice of 2: operator and version
+// and builds a versionRange, otherwise an error.
+func buildVersionRange(opStr, vStr string) (*versionRange, error) {
+	c := parseComparator(opStr)
+	if c == nil {
+		return nil, fmt.Errorf("Could not parse comparator %q in %q", opStr, strings.Join([]string{opStr, vStr}, ""))
+	}
+	v, err := Parse(vStr)
+	if err != nil {
+		return nil, fmt.Errorf("Could not parse version %q in %q: %s", vStr, strings.Join([]string{opStr, vStr}, ""), err)
+	}
+
+	return &versionRange{
+		v: v,
+		c: c,
+	}, nil
+
+}
+
+// inArray checks if a byte is contained in an array of bytes
+func inArray(s byte, list []byte) bool {
+	for _, el := range list {
+		if el == s {
+			return true
+		}
+	}
+	return false
+}
+
+// splitAndTrim splits a range string by spaces and cleans whitespaces
+func splitAndTrim(s string) (result []string) {
+	last := 0
+	var lastChar byte
+	excludeFromSplit := []byte{'>', '<', '='}
+	for i := 0; i < len(s); i++ {
+		if s[i] == ' ' && !inArray(lastChar, excludeFromSplit) {
+			if last < i-1 {
+				result = append(result, s[last:i])
+			}
+			last = i + 1
+		} else if s[i] != ' ' {
+			lastChar = s[i]
+		}
+	}
+	if last < len(s)-1 {
+		result = append(result, s[last:])
+	}
+
+	for i, v := range result {
+		result[i] = strings.Replace(v, " ", "", -1)
+	}
+
+	// parts := strings.Split(s, " ")
+	// for _, x := range parts {
+	// 	if s := strings.TrimSpace(x); len(s) != 0 {
+	// 		result = append(result, s)
+	// 	}
+	// }
+	return
+}
+
+// splitComparatorVersion splits the comparator from the version.
+// Input must be free of leading or trailing spaces.
+func splitComparatorVersion(s string) (string, string, error) {
+	i := strings.IndexFunc(s, unicode.IsDigit)
+	if i == -1 {
+		return "", "", fmt.Errorf("Could not get version from string: %q", s)
+	}
+	return strings.TrimSpace(s[0:i]), s[i:], nil
+}
+
+// getWildcardType will return the type of wildcard that the
+// passed version contains
+func getWildcardType(vStr string) wildcardType {
+	parts := strings.Split(vStr, ".")
+	nparts := len(parts)
+	wildcard := parts[nparts-1]
+
+	possibleWildcardType := wildcardTypefromInt(nparts)
+	if wildcard == "x" {
+		return possibleWildcardType
+	}
+
+	return noneWildcard
+}
+
+// createVersionFromWildcard will convert a wildcard version
+// into a regular version, replacing 'x's with '0's, handling
+// special cases like '1.x.x' and '1.x'
+func createVersionFromWildcard(vStr string) string {
+	// handle 1.x.x
+	vStr2 := strings.Replace(vStr, ".x.x", ".x", 1)
+	vStr2 = strings.Replace(vStr2, ".x", ".0", 1)
+	parts := strings.Split(vStr2, ".")
+
+	// handle 1.x
+	if len(parts) == 2 {
+		return vStr2 + ".0"
+	}
+
+	return vStr2
+}
+
+// incrementMajorVersion will increment the major version
+// of the passed version
+func incrementMajorVersion(vStr string) (string, error) {
+	parts := strings.Split(vStr, ".")
+	i, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return "", err
+	}
+	parts[0] = strconv.Itoa(i + 1)
+
+	return strings.Join(parts, "."), nil
+}
+
+// incrementMajorVersion will increment the minor version
+// of the passed version
+func incrementMinorVersion(vStr string) (string, error) {
+	parts := strings.Split(vStr, ".")
+	i, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return "", err
+	}
+	parts[1] = strconv.Itoa(i + 1)
+
+	return strings.Join(parts, "."), nil
+}
+
+// expandWildcardVersion will expand wildcards inside versions
+// following these rules:
+//
+// * when dealing with patch wildcards:
+// >= 1.2.x    will become    >= 1.2.0
+// <= 1.2.x    will become    <  1.3.0
+// >  1.2.x    will become    >= 1.3.0
+// <  1.2.x    will become    <  1.2.0
+// != 1.2.x    will become    <  1.2.0 >= 1.3.0
+//
+// * when dealing with minor wildcards:
+// >= 1.x      will become    >= 1.0.0
+// <= 1.x      will become    <  2.0.0
+// >  1.x      will become    >= 2.0.0
+// <  1.0      will become    <  1.0.0
+// != 1.x      will become    <  1.0.0 >= 2.0.0
+//
+// * when dealing with wildcards without
+// version operator:
+// 1.2.x       will become    >= 1.2.0 < 1.3.0
+// 1.x         will become    >= 1.0.0 < 2.0.0
+func expandWildcardVersion(parts [][]string) ([][]string, error) {
+	var expandedParts [][]string
+	for _, p := range parts {
+		var newParts []string
+		for _, ap := range p {
+			if strings.Index(ap, "x") != -1 {
+				opStr, vStr, err := splitComparatorVersion(ap)
+				if err != nil {
+					return nil, err
+				}
+
+				versionWildcardType := getWildcardType(vStr)
+				flatVersion := createVersionFromWildcard(vStr)
+
+				var resultOperator string
+				var shouldIncrementVersion bool
+				switch opStr {
+				case ">":
+					resultOperator = ">="
+					shouldIncrementVersion = true
+				case ">=":
+					resultOperator = ">="
+				case "<":
+					resultOperator = "<"
+				case "<=":
+					resultOperator = "<"
+					shouldIncrementVersion = true
+				case "", "=", "==":
+					newParts = append(newParts, ">="+flatVersion)
+					resultOperator = "<"
+					shouldIncrementVersion = true
+				case "!=", "!":
+					newParts = append(newParts, "<"+flatVersion)
+					resultOperator = ">="
+					shouldIncrementVersion = true
+				}
+
+				var resultVersion string
+				if shouldIncrementVersion {
+					switch versionWildcardType {
+					case patchWildcard:
+						resultVersion, _ = incrementMinorVersion(flatVersion)
+					case minorWildcard:
+						resultVersion, _ = incrementMajorVersion(flatVersion)
+					}
+				} else {
+					resultVersion = flatVersion
+				}
+
+				ap = resultOperator + resultVersion
+			}
+			newParts = append(newParts, ap)
+		}
+		expandedParts = append(expandedParts, newParts)
+	}
+
+	return expandedParts, nil
+}
+
+func parseComparator(s string) comparator {
+	switch s {
+	case "==":
+		fallthrough
+	case "":
+		fallthrough
+	case "=":
+		return compEQ
+	case ">":
+		return compGT
+	case ">=":
+		return compGE
+	case "<":
+		return compLT
+	case "<=":
+		return compLE
+	case "!":
+		fallthrough
+	case "!=":
+		return compNE
+	}
+
+	return nil
+}
+
+// MustParseRange is like ParseRange but panics if the range cannot be parsed.
+func MustParseRange(s string) Range {
+	r, err := ParseRange(s)
+	if err != nil {
+		panic(`semver: ParseRange(` + s + `): ` + err.Error())
+	}
+	return r
+}

vendor/github.com/blang/semver/semver.go

@@ -0,0 +1,418 @@
+package semver
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const (
+	numbers  string = "0123456789"
+	alphas          = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
+	alphanum        = alphas + numbers
+)
+
+// SpecVersion is the latest fully supported spec version of semver
+var SpecVersion = Version{
+	Major: 2,
+	Minor: 0,
+	Patch: 0,
+}
+
+// Version represents a semver compatible version
+type Version struct {
+	Major uint64
+	Minor uint64
+	Patch uint64
+	Pre   []PRVersion
+	Build []string //No Precendence
+}
+
+// Version to string
+func (v Version) String() string {
+	b := make([]byte, 0, 5)
+	b = strconv.AppendUint(b, v.Major, 10)
+	b = append(b, '.')
+	b = strconv.AppendUint(b, v.Minor, 10)
+	b = append(b, '.')
+	b = strconv.AppendUint(b, v.Patch, 10)
+
+	if len(v.Pre) > 0 {
+		b = append(b, '-')
+		b = append(b, v.Pre[0].String()...)
+
+		for _, pre := range v.Pre[1:] {
+			b = append(b, '.')
+			b = append(b, pre.String()...)
+		}
+	}
+
+	if len(v.Build) > 0 {
+		b = append(b, '+')
+		b = append(b, v.Build[0]...)
+
+		for _, build := range v.Build[1:] {
+			b = append(b, '.')
+			b = append(b, build...)
+		}
+	}
+
+	return string(b)
+}
+
+// Equals checks if v is equal to o.
+func (v Version) Equals(o Version) bool {
+	return (v.Compare(o) == 0)
+}
+
+// EQ checks if v is equal to o.
+func (v Version) EQ(o Version) bool {
+	return (v.Compare(o) == 0)
+}
+
+// NE checks if v is not equal to o.
+func (v Version) NE(o Version) bool {
+	return (v.Compare(o) != 0)
+}
+
+// GT checks if v is greater than o.
+func (v Version) GT(o Version) bool {
+	return (v.Compare(o) == 1)
+}
+
+// GTE checks if v is greater than or equal to o.
+func (v Version) GTE(o Version) bool {
+	return (v.Compare(o) >= 0)
+}
+
+// GE checks if v is greater than or equal to o.
+func (v Version) GE(o Version) bool {
+	return (v.Compare(o) >= 0)
+}
+
+// LT checks if v is less than o.
+func (v Version) LT(o Version) bool {
+	return (v.Compare(o) == -1)
+}
+
+// LTE checks if v is less than or equal to o.
+func (v Version) LTE(o Version) bool {
+	return (v.Compare(o) <= 0)
+}
+
+// LE checks if v is less than or equal to o.
+func (v Version) LE(o Version) bool {
+	return (v.Compare(o) <= 0)
+}
+
+// Compare compares Versions v to o:
+// -1 == v is less than o
+// 0 == v is equal to o
+// 1 == v is greater than o
+func (v Version) Compare(o Version) int {
+	if v.Major != o.Major {
+		if v.Major > o.Major {
+			return 1
+		}
+		return -1
+	}
+	if v.Minor != o.Minor {
+		if v.Minor > o.Minor {
+			return 1
+		}
+		return -1
+	}
+	if v.Patch != o.Patch {
+		if v.Patch > o.Patch {
+			return 1
+		}
+		return -1
+	}
+
+	// Quick comparison if a version has no prerelease versions
+	if len(v.Pre) == 0 && len(o.Pre) == 0 {
+		return 0
+	} else if len(v.Pre) == 0 && len(o.Pre) > 0 {
+		return 1
+	} else if len(v.Pre) > 0 && len(o.Pre) == 0 {
+		return -1
+	}
+
+	i := 0
+	for ; i < len(v.Pre) && i < len(o.Pre); i++ {
+		if comp := v.Pre[i].Compare(o.Pre[i]); comp == 0 {
+			continue
+		} else if comp == 1 {
+			return 1
+		} else {
+			return -1
+		}
+	}
+
+	// If all pr versions are the equal but one has further prversion, this one greater
+	if i == len(v.Pre) && i == len(o.Pre) {
+		return 0
+	} else if i == len(v.Pre) && i < len(o.Pre) {
+		return -1
+	} else {
+		return 1
+	}
+
+}
+
+// Validate validates v and returns error in case
+func (v Version) Validate() error {
+	// Major, Minor, Patch already validated using uint64
+
+	for _, pre := range v.Pre {
+		if !pre.IsNum { //Numeric prerelease versions already uint64
+			if len(pre.VersionStr) == 0 {
+				return fmt.Errorf("Prerelease can not be empty %q", pre.VersionStr)
+			}
+			if !containsOnly(pre.VersionStr, alphanum) {
+				return fmt.Errorf("Invalid character(s) found in prerelease %q", pre.VersionStr)
+			}
+		}
+	}
+
+	for _, build := range v.Build {
+		if len(build) == 0 {
+			return fmt.Errorf("Build meta data can not be empty %q", build)
+		}
+		if !containsOnly(build, alphanum) {
+			return fmt.Errorf("Invalid character(s) found in build meta data %q", build)
+		}
+	}
+
+	return nil
+}
+
+// New is an alias for Parse and returns a pointer, parses version string and returns a validated Version or error
+func New(s string) (vp *Version, err error) {
+	v, err := Parse(s)
+	vp = &v
+	return
+}
+
+// Make is an alias for Parse, parses version string and returns a validated Version or error
+func Make(s string) (Version, error) {
+	return Parse(s)
+}
+
+// ParseTolerant allows for certain version specifications that do not strictly adhere to semver
+// specs to be parsed by this library. It does so by normalizing versions before passing them to
+// Parse(). It currently trims spaces, removes a "v" prefix, and adds a 0 patch number to versions
+// with only major and minor components specified
+func ParseTolerant(s string) (Version, error) {
+	s = strings.TrimSpace(s)
+	s = strings.TrimPrefix(s, "v")
+
+	// Split into major.minor.(patch+pr+meta)
+	parts := strings.SplitN(s, ".", 3)
+	if len(parts) < 3 {
+		if strings.ContainsAny(parts[len(parts)-1], "+-") {
+			return Version{}, errors.New("Short version cannot contain PreRelease/Build meta data")
+		}
+		for len(parts) < 3 {
+			parts = append(parts, "0")
+		}
+		s = strings.Join(parts, ".")
+	}
+
+	return Parse(s)
+}
+
+// Parse parses version string and returns a validated Version or error
+func Parse(s string) (Version, error) {
+	if len(s) == 0 {
+		return Version{}, errors.New("Version string empty")
+	}
+
+	// Split into major.minor.(patch+pr+meta)
+	parts := strings.SplitN(s, ".", 3)
+	if len(parts) != 3 {
+		return Version{}, errors.New("No Major.Minor.Patch elements found")
+	}
+
+	// Major
+	if !containsOnly(parts[0], numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in major number %q", parts[0])
+	}
+	if hasLeadingZeroes(parts[0]) {
+		return Version{}, fmt.Errorf("Major number must not contain leading zeroes %q", parts[0])
+	}
+	major, err := strconv.ParseUint(parts[0], 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	// Minor
+	if !containsOnly(parts[1], numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in minor number %q", parts[1])
+	}
+	if hasLeadingZeroes(parts[1]) {
+		return Version{}, fmt.Errorf("Minor number must not contain leading zeroes %q", parts[1])
+	}
+	minor, err := strconv.ParseUint(parts[1], 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	v := Version{}
+	v.Major = major
+	v.Minor = minor
+
+	var build, prerelease []string
+	patchStr := parts[2]
+
+	if buildIndex := strings.IndexRune(patchStr, '+'); buildIndex != -1 {
+		build = strings.Split(patchStr[buildIndex+1:], ".")
+		patchStr = patchStr[:buildIndex]
+	}
+
+	if preIndex := strings.IndexRune(patchStr, '-'); preIndex != -1 {
+		prerelease = strings.Split(patchStr[preIndex+1:], ".")
+		patchStr = patchStr[:preIndex]
+	}
+
+	if !containsOnly(patchStr, numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in patch number %q", patchStr)
+	}
+	if hasLeadingZeroes(patchStr) {
+		return Version{}, fmt.Errorf("Patch number must not contain leading zeroes %q", patchStr)
+	}
+	patch, err := strconv.ParseUint(patchStr, 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	v.Patch = patch
+
+	// Prerelease
+	for _, prstr := range prerelease {
+		parsedPR, err := NewPRVersion(prstr)
+		if err != nil {
+			return Version{}, err
+		}
+		v.Pre = append(v.Pre, parsedPR)
+	}
+
+	// Build meta data
+	for _, str := range build {
+		if len(str) == 0 {
+			return Version{}, errors.New("Build meta data is empty")
+		}
+		if !containsOnly(str, alphanum) {
+			return Version{}, fmt.Errorf("Invalid character(s) found in build meta data %q", str)
+		}
+		v.Build = append(v.Build, str)
+	}
+
+	return v, nil
+}
+
+// MustParse is like Parse but panics if the version cannot be parsed.
+func MustParse(s string) Version {
+	v, err := Parse(s)
+	if err != nil {
+		panic(`semver: Parse(` + s + `): ` + err.Error())
+	}
+	return v
+}
+
+// PRVersion represents a PreRelease Version
+type PRVersion struct {
+	VersionStr string
+	VersionNum uint64
+	IsNum      bool
+}
+
+// NewPRVersion creates a new valid prerelease version
+func NewPRVersion(s string) (PRVersion, error) {
+	if len(s) == 0 {
+		return PRVersion{}, errors.New("Prerelease is empty")
+	}
+	v := PRVersion{}
+	if containsOnly(s, numbers) {
+		if hasLeadingZeroes(s) {
+			return PRVersion{}, fmt.Errorf("Numeric PreRelease version must not contain leading zeroes %q", s)
+		}
+		num, err := strconv.ParseUint(s, 10, 64)
+
+		// Might never be hit, but just in case
+		if err != nil {
+			return PRVersion{}, err
+		}
+		v.VersionNum = num
+		v.IsNum = true
+	} else if containsOnly(s, alphanum) {
+		v.VersionStr = s
+		v.IsNum = false
+	} else {
+		return PRVersion{}, fmt.Errorf("Invalid character(s) found in prerelease %q", s)
+	}
+	return v, nil
+}
+
+// IsNumeric checks if prerelease-version is numeric
+func (v PRVersion) IsNumeric() bool {
+	return v.IsNum
+}
+
+// Compare compares two PreRelease Versions v and o:
+// -1 == v is less than o
+// 0 == v is equal to o
+// 1 == v is greater than o
+func (v PRVersion) Compare(o PRVersion) int {
+	if v.IsNum && !o.IsNum {
+		return -1
+	} else if !v.IsNum && o.IsNum {
+		return 1
+	} else if v.IsNum && o.IsNum {
+		if v.VersionNum == o.VersionNum {
+			return 0
+		} else if v.VersionNum > o.VersionNum {
+			return 1
+		} else {
+			return -1
+		}
+	} else { // both are Alphas
+		if v.VersionStr == o.VersionStr {
+			return 0
+		} else if v.VersionStr > o.VersionStr {
+			return 1
+		} else {
+			return -1
+		}
+	}
+}
+
+// PreRelease version to string
+func (v PRVersion) String() string {
+	if v.IsNum {
+		return strconv.FormatUint(v.VersionNum, 10)
+	}
+	return v.VersionStr
+}
+
+func containsOnly(s string, set string) bool {
+	return strings.IndexFunc(s, func(r rune) bool {
+		return !strings.ContainsRune(set, r)
+	}) == -1
+}
+
+func hasLeadingZeroes(s string) bool {
+	return len(s) > 1 && s[0] == '0'
+}
+
+// NewBuildVersion creates a new valid build version
+func NewBuildVersion(s string) (string, error) {
+	if len(s) == 0 {
+		return "", errors.New("Buildversion is empty")
+	}
+	if !containsOnly(s, alphanum) {
+		return "", fmt.Errorf("Invalid character(s) found in build meta data %q", s)
+	}
+	return s, nil
+}

vendor/github.com/blang/semver/sort.go

@@ -0,0 +1,28 @@
+package semver
+
+import (
+	"sort"
+)
+
+// Versions represents multiple versions.
+type Versions []Version
+
+// Len returns length of version collection
+func (s Versions) Len() int {
+	return len(s)
+}
+
+// Swap swaps two versions inside the collection by its indices
+func (s Versions) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+// Less checks if version at index i is less than version at index j
+func (s Versions) Less(i, j int) bool {
+	return s[i].LT(s[j])
+}
+
+// Sort sorts a slice of versions
+func Sort(versions []Version) {
+	sort.Sort(Versions(versions))
+}

vendor/github.com/blang/semver/sql.go

@@ -0,0 +1,30 @@
+package semver
+
+import (
+	"database/sql/driver"
+	"fmt"
+)
+
+// Scan implements the database/sql.Scanner interface.
+func (v *Version) Scan(src interface{}) (err error) {
+	var str string
+	switch src := src.(type) {
+	case string:
+		str = src
+	case []byte:
+		str = string(src)
+	default:
+		return fmt.Errorf("Version.Scan: cannot convert %T to string.", src)
+	}
+
+	if t, err := Parse(str); err == nil {
+		*v = t
+	}
+
+	return
+}
+
+// Value implements the database/sql/driver.Valuer interface.
+func (v Version) Value() (driver.Value, error) {
+	return v.String(), nil
+}

vendor/github.com/cespare/xxhash/v2/.travis.yml

@@ -0,0 +1,8 @@
+language: go
+go:
+  - "1.x"
+  - master
+env:
+  - TAGS=""
+  - TAGS="-tags purego"
+script: go test $TAGS -v ./...

vendor/github.com/cespare/xxhash/v2/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Caleb Spare
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/cespare/xxhash/v2/README.md

@@ -0,0 +1,67 @@
+# xxhash
+
+[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
+[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash)
+
+xxhash is a Go implementation of the 64-bit
+[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
+high-quality hashing algorithm that is much faster than anything in the Go
+standard library.
+
+This package provides a straightforward API:
+
+```
+func Sum64(b []byte) uint64
+func Sum64String(s string) uint64
+type Digest struct{ ... }
+    func New() *Digest
+```
+
+The `Digest` type implements hash.Hash64. Its key methods are:
+
+```
+func (*Digest) Write([]byte) (int, error)
+func (*Digest) WriteString(string) (int, error)
+func (*Digest) Sum64() uint64
+```
+
+This implementation provides a fast pure-Go implementation and an even faster
+assembly implementation for amd64.
+
+## Compatibility
+
+This package is in a module and the latest code is in version 2 of the module.
+You need a version of Go with at least "minimal module compatibility" to use
+github.com/cespare/xxhash/v2:
+
+* 1.9.7+ for Go 1.9
+* 1.10.3+ for Go 1.10
+* Go 1.11 or later
+
+I recommend using the latest release of Go.
+
+## Benchmarks
+
+Here are some quick benchmarks comparing the pure-Go and assembly
+implementations of Sum64.
+
+| input size | purego | asm |
+| --- | --- | --- |
+| 5 B   |  979.66 MB/s |  1291.17 MB/s  |
+| 100 B | 7475.26 MB/s | 7973.40 MB/s  |
+| 4 KB  | 17573.46 MB/s | 17602.65 MB/s |
+| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
+
+These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
+the following commands under Go 1.11.2:
+
+```
+$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
+$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
+```
+
+## Projects using this package
+
+- [InfluxDB](https://github.com/influxdata/influxdb)
+- [Prometheus](https://github.com/prometheus/prometheus)
+- [FreeCache](https://github.com/coocood/freecache)

vendor/github.com/cespare/xxhash/v2/go.mod

@@ -0,0 +1,3 @@
+module github.com/cespare/xxhash/v2
+
+go 1.11