Fix github action

This change update goleleaser action and goreleaser config file
to release multus binaries, including multus-daemon and multus-shim

.github/workflows/build.yml

@@ -4,18 +4,18 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.19.x, 1.20.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Build
       env:

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: "46 8 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ go ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/image-build.yml

@@ -1,128 +1,76 @@
 name: Image build
 on: [pull_request]
 jobs:
-  ep-build-amd64:
-    name: Image build/amd64 LEGACY entrypoint
+  build-thin:
+    name: Image build thin plugin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
+      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:ep-latest-amd64
+          tags: ghcr.io/${{ github.repository }}:latest
           file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
-  build-amd64:
-    name: Image build/amd64 daemonized alternative
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
+      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
+      - name: Build container debug image
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:latest-amd64
+          tags: ghcr.io/${{ github.repository }}:latest
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+  build-amd64-thick:
+    name: Image build/amd64 thick plugin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build container image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-amd64-thick
           file: images/Dockerfile.thick
 
-  build-arm64:
-    name: Image build/arm64
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-arm64
-          file: images/Dockerfile.arm64
-
-  build-arm32:
-    name: Image build/arm32
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-arm32
-          file: images/Dockerfile.arm32
-
-  build-ppc64le:
-    name: Image build/ppc64le
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-ppc64le
-          file: images/Dockerfile.ppc64le
-
-  build-s390:
-    name: Image build/s390x
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-s390x
-          file: images/Dockerfile.s390x
-
   build-origin:
     name: Image build/origin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
-      - name: Build container image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest-origin
-          file: images/Dockerfile.openshift
+      - name: Download OKD Builder Dockerfile
+        run: curl https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile -o images/okd-builder.Dockerfile
+
+      - name: Patch OKD Builder Dockerfile to workaround error
+        run: sed -i -e "s/yum install -y yum-utils/rpm --import \/etc\/pki\/rpm-gpg\/*;yum install -y yum-utils/" images/okd-builder.Dockerfile
+
+      - name: Organically build golang builder image
+        run: docker build -t local/okdbuilder:latest -f images/okd-builder.Dockerfile .
+
+      - name: Organically build Multus origin image
+        run: docker build -t local/multus-cni:latest-origin -f images/Dockerfile.openshift .

.github/workflows/image-push-master.yml

@@ -3,238 +3,112 @@ on:
   push:
     branches:
       - master
+env:
+  image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-amd64:
-    name: Image push/amd64
+  push-thick-amd64:
+    name: Image push thick image/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push container image for thick plugin
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:latest-amd64
-            ghcr.io/${{ github.repository }}:snapshot-amd64
-          file: images/Dockerfile
-
-      - name: Push container image for daemon based deployment
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:thick-amd64
+            ghcr.io/${{ github.repository }}:latest-thick
+            ghcr.io/${{ github.repository }}:snapshot-thick
           file: images/Dockerfile.thick
+          platforms: linux/amd64
 
-  push-arm64:
-    name: Image push/arm64
+  push-thin:
+    name: Image push thin image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push thin container image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:latest-arm64
-            ghcr.io/${{ github.repository }}:snapshot-arm64
-          file: images/Dockerfile.arm64
+            ghcr.io/${{ github.repository }}:latest
+            ghcr.io/${{ github.repository }}:snapshot
+          file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
-  push-arm32:
-    name: Image push/arm32
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push thin container debug image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:latest-arm32
-            ghcr.io/${{ github.repository }}:snapshot-arm32
-          file: images/Dockerfile.arm32
+            ghcr.io/${{ github.repository }}:latest-debug
+            ghcr.io/${{ github.repository }}:snapshot-debug
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
-  push-ppc64le:
-    name: Image push/ppc64le
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+# TODO: need to fix this action
+#  push-origin:
+#    name: Image push/origin
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v3
+#
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v2
+#
+#      - name: Login to GitHub Container Registry
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/login-action@v2
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Push container image
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/build-push-action@v3
+#        with:
+#          context: .
+#          push: true
+#          tags: |
+#            ghcr.io/${{ github.repository }}:latest-origin
+#            ghcr.io/${{ github.repository }}:snapshot-origin
+#          file: images/Dockerfile.openshift
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:latest-ppc64le
-            ghcr.io/${{ github.repository }}:snapshot-ppc64le
-          file: images/Dockerfile.ppc64le
-
-  push-s390x:
-    name: Image push/s390x
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:latest-s390x
-            ghcr.io/${{ github.repository }}:snapshot-s390x
-          file: images/Dockerfile.s390x
-
-  push-origin:
-    name: Image push/origin
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:latest-origin
-            ghcr.io/${{ github.repository }}:snapshot-origin
-          file: images/Dockerfile.openshift
-
-  push-manifest:
-    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
-    runs-on: ubuntu-latest
-    env:
-      REPOSITORY: ghcr.io/${{ github.repository }}
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create manifest for multi-arch images
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        run: |
-          # get artifacts from previous steps
-          docker pull ${{ env.REPOSITORY }}:thick-amd64
-          docker manifest create ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64
-          docker manifest annotate ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64 --arch amd64
-          docker manifest push ${{ env.REPOSITORY }}:thick
-
-          docker pull ${{ env.REPOSITORY }}:snapshot-amd64
-          docker pull ${{ env.REPOSITORY }}:snapshot-arm64
-          docker pull ${{ env.REPOSITORY }}:snapshot-arm32
-          docker pull ${{ env.REPOSITORY }}:snapshot-ppc64le
-          docker pull ${{ env.REPOSITORY }}:snapshot-s390x
-          docker pull ${{ env.REPOSITORY }}:latest-amd64
-          docker pull ${{ env.REPOSITORY }}:latest-arm64
-          docker pull ${{ env.REPOSITORY }}:latest-arm32
-          docker pull ${{ env.REPOSITORY }}:latest-ppc64le
-          docker pull ${{ env.REPOSITORY }}:latest-s390x
-          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-arm32 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
-          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 --arch amd64
-          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm64 --arch arm64
-          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm32 --arch arm
-          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-ppc64le --arch ppc64le
-          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-s390x --arch s390x
-          docker manifest push ${{ env.REPOSITORY }}:snapshot
-          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-arm32 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
-          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 --arch amd64
-          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm64 --arch arm64
-          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm32 --arch arm
-          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-ppc64le --arch ppc64le
-          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-s390x --arch s390x
-          docker manifest push ${{ env.REPOSITORY }}:latest

.github/workflows/image-push-release.yml

@@ -3,20 +3,22 @@ on:
   push:
     tags:
       - v*
+env:
+  image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-amd64:
-    name: Image push/amd64
+  push-thick-amd64:
+    name: Image push thick image/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -24,46 +26,37 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
-          tag-latest: false
+          flavor: |
+            latest=false
 
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push container image for thick plugin
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:stable-amd64
-            ${{ steps.docker_meta.outputs.tags }}-amd64
-          file: images/Dockerfile
-
-      - name: Push container image for daemon based deployment
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:thick-amd64
-            ${{ steps.docker_meta.outputs.tags }}-thick-amd64
+            ghcr.io/${{ github.repository }}:stable-thick
+            ${{ steps.docker_meta.outputs.tags }}-thick
           file: images/Dockerfile.thick
+          platforms: linux/amd64
 
-  push-arm64:
-    name: Image push/arm64
+  push-thin:
+    name: Image push thin image/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -71,215 +64,73 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
-          tag-latest: false
+          flavor: |
+            latest=false
 
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push thin container image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:stable-arm64
-            ${{ steps.docker_meta.outputs.tags }}-arm64
-          file: images/Dockerfile.arm64
+            ghcr.io/${{ github.repository }}:stable
+            ${{ steps.docker_meta.outputs.tags }}
+          file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
-  push-arm32:
-    name: Image push/arm32
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tag-latest: false
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
+      - name: Push thin container debug image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:stable-arm32
-            ${{ steps.docker_meta.outputs.tags }}-arm32
-          file: images/Dockerfile.arm32
+            ghcr.io/${{ github.repository }}:stable-debug
+            ${{ steps.docker_meta.outputs.tags }}-debug
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
-  push-ppc64le:
-    name: Image push/ppc64le
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tag-latest: false
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:stable-ppc64le
-            ${{ steps.docker_meta.outputs.tags }}-ppc64le
-          file: images/Dockerfile.ppc64le
-
-  push-s390x:
-    name: Image push/s390x
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tag-latest: false
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:stable-s390x
-            ${{ steps.docker_meta.outputs.tags }}-s390x
-          file: images/Dockerfile.s390x
-
-  push-origin:
-    name: Image push/origin
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tag-latest: false
-
-      - name: Push container image
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:stable-origin
-            ${{ steps.docker_meta.outputs.tags }}-origin
-          file: images/Dockerfile.openshift
-
-  push-manifest:
-    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
-    runs-on: ubuntu-latest
-    env:
-      REPOSITORY: ghcr.io/${{ github.repository }}
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tag-latest: false
-
-      - name: Login to GitHub Container Registry
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create manifest for multi-arch images
-        if: github.repository_owner == 'k8snetworkplumbingwg'
-        run: |
-          # get artifacts from previous steps
-          docker pull ${{ steps.docker_meta.outputs.tags }}-amd64
-          docker pull ${{ steps.docker_meta.outputs.tags }}-arm64
-          docker pull ${{ steps.docker_meta.outputs.tags }}-arm32
-          docker pull ${{ steps.docker_meta.outputs.tags }}-ppc64le
-          docker pull ${{ steps.docker_meta.outputs.tags }}-s390x
-          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-arm32 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
-          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 --arch amd64
-          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm64 --arch arm64
-          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm32 --arch arm
-          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-ppc64le --arch ppc64le
-          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-s390x --arch s390x
-          docker manifest push ${{ steps.docker_meta.outputs.tags }}
-          docker pull ${{ env.REPOSITORY }}:stable-amd64
-          docker pull ${{ env.REPOSITORY }}:stable-arm64
-          docker pull ${{ env.REPOSITORY }}:stable-arm32
-          docker pull ${{ env.REPOSITORY }}:stable-ppc64le
-          docker pull ${{ env.REPOSITORY }}:stable-s390x
-          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-arm32 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
-          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 --arch amd64
-          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm64 --arch arm64
-          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm32 --arch arm
-          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-ppc64le --arch ppc64le
-          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-s390x --arch s390x
-          docker manifest push ${{ env.REPOSITORY }}:stable
+# TODO: need to fix this action
+#  push-origin:
+#    name: Image push/origin
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v3
+#
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v1
+#
+#      - name: Login to GitHub Container Registry
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/login-action@v1
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Docker meta
+#        id: docker_meta
+#        uses: crazy-max/ghaction-docker-meta@v1
+#        with:
+#          images: ghcr.io/${{ github.repository }}
+#          tag-latest: false
+#
+#      - name: Push container image
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/build-push-action@v2
+#        with:
+#          context: .
+#          push: true
+#          tags: |
+#            ghcr.io/${{ github.repository }}:stable-origin
+#            ${{ steps.docker_meta.outputs.tags }}-origin
+#          file: images/Dockerfile.openshift

.github/workflows/kind-e2e.yml

@@ -3,19 +3,62 @@ on: [push, pull_request]
 jobs:
   e2e-kind:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - docker-file: images/Dockerfile.thick
+            cni-version: "0.3.1"
+            multus-manifest: multus-daemonset-thick.yml
+          - docker-file: images/Dockerfile
+            cni-version: "0.3.1"
+            multus-manifest: multus-daemonset.yml
+          - docker-file: images/Dockerfile.thick
+            cni-version: "0.4.0"
+            multus-manifest: multus-daemonset-thick.yml
+          - docker-file: images/Dockerfile
+            cni-version: "0.4.0"
+            multus-manifest: multus-daemonset.yml
+          # need to wait kind to support CNI 1.0.0 (now kind 0.11 supports up to 0.4.0)
+#          - docker-file: images/Dockerfile.thick
+#            cni-version: "1.0.0"
+#            multus-manifest: multus-thick-daemonset.yml
+#          - docker-file: images/Dockerfile
+#            cni-version: "1.0.0"
+#            multus-manifest: multus-daemonset.yml
+
     if: >
       (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
         github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+
+      - name: Setup j2cli
+        run: |
+          pip3 install --user --upgrade j2cli
+          j2 --version
 
       - name: Setup registry
         run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
-  
+
       - name: Build latest-amd64
-        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile.thick .
- 
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          load: true
+          tags: localhost:5000/multus:e2e
+          file: ${{ matrix.docker-file }}
+          platforms: linux/amd64
+
+      # docker buildx push is failed due to https://github.com/docker/buildx/issues/94
       - name: Push to local registry
         run: docker push localhost:5000/multus:e2e
 
@@ -23,9 +66,13 @@ jobs:
         working-directory: ./e2e
         run: ./get_tools.sh
 
+      - name: generate yaml files
+        working-directory: ./e2e
+        run: env CNI_VERSION=${{ matrix.cni-version }} ./generate_yamls.sh
+
       - name: Setup cluster
         working-directory: ./e2e
-        run: ./setup_cluster.sh
+        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} ./setup_cluster.sh
 
       - name: Test simple pod
         working-directory: ./e2e

.github/workflows/legacy-kind-e2e.yml

@@ -1,46 +0,0 @@
-name: e2e-kind legacy installation with entrypoint script
-on: [push, pull_request]
-jobs:
-  e2e-kind:
-    runs-on: ubuntu-latest
-    if: >
-      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
-        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Setup registry
-        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
-
-      - name: Build latest-amd64
-        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile .
-
-      - name: Push to local registry
-        run: docker push localhost:5000/multus:e2e
-
-      - name: Get kind/kubectl/koko
-        working-directory: ./e2e
-        run: ./get_tools.sh
-
-      - name: Setup cluster
-        working-directory: ./e2e
-        run: MULTUS_MANIFEST=legacy-multus-daemonset.yml ./setup_cluster.sh
-
-      - name: Test simple pod
-        working-directory: ./e2e
-        run: ./test-simple-pod.sh
-
-      - name: Test macvlan1
-        working-directory: ./e2e
-        run: ./test-simple-macvlan1.sh
-
-      - name: Test default route1
-        working-directory: ./e2e
-        run: ./test-default-route1.sh
-
-      - name: cleanup cluster and registry
-        run: |
-          kind delete cluster
-          docker kill kind-registry
-          docker rm kind-registry

.github/workflows/release.yml

@@ -8,17 +8,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17.x
+          go-version: 1.19.x
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: latest
           args: release --rm-dist

.github/workflows/test.yml

@@ -4,23 +4,22 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.19.x, 1.20.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - name: Run Revive
-      run: |
-        GO111MODULE=off go get github.com/mgechev/revive
-        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./... # this is ouput for user
-        $(go env GOPATH)/bin/revive -exclude ./vendor/... ./...| xargs -0 -r false # this is for github actions
+    - name: Run Revive Action by pulling pre-built image
+      uses: docker://morphy/revive-action:v2
+      with:
+        exclude: "./vendor/..."
 
     - name: Run go fmt
       run: go fmt ./...

.gitignore

@@ -1,6 +1,7 @@
 # Binary output dir
 bin/
 e2e/bin/
+e2e/yamls/
 
 # GOPATH created by the build script
 gopath/

.goreleaser.yml

@@ -6,20 +6,51 @@ before:
   hooks:
     - go mod download
 builds:
--  
-  env:
-  - CGO_ENABLED=0
-  main: ./cmd/
-  goos:
-  - linux
-  goarch:
-  - 386
-  - amd64
-  - arm
-  - arm64
-  - s390x
-  ldflags:
-  - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date={{ .Date }}
+  - env:
+      - CGO_ENABLED=0
+    id: multus
+    binary: multus
+    main: ./cmd/multus
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+  - env:
+      - CGO_ENABLED=0
+    id: multus-daemon
+    binary: multus-daemon
+    main: ./cmd/multus-daemon
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+  - env:
+      - CGO_ENABLED=0
+    id: multus-shim
+    binary: multus-shim
+    main: ./cmd/multus-shim
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
 archives:
   - wrap_in_directory: true
 checksum:

NOTICE

@@ -0,0 +1,2 @@
+Copyright 2016 Intel Corporation
+Copyright 2021 Multus Authors

README.md

@@ -22,30 +22,43 @@ Here's an illustration of the network interfaces attached to a pod, as provision
 
 ## Quickstart Installation Guide
 
-The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugin, refer to our [quick-start guide](docs/quickstart.md).
 
-Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
+Clone this GitHub repository, and apply a daemonset which installs Multus using `kubectl`. From the root directory of the clone, apply the daemonset YAML file:
 
 ```
-cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
 
-## Additional installation Options
+## Thin Plugin v.s Thick Plugin
+
+With the multus 4.0 release, we introduce a new client/server-style plugin deployment. This new deployment is called ['thick plugin'](docs/thick-plugin.md), in contrast to deployments in previous versions, which is now called a 'thin plugin'. The new thick plugin consists of two binaries, multus-daemon and multus-shim CNI plugin. The 'multus-daemon' will be deployed to all nodes as a local agent and supports additional features, such as metrics, which were not available with the 'thin plugin' deployment before. Due to these additional features, the 'thick plugin' comes with the trade-off of consuming more resources than the 'thin plugin'.
+
+We recommend using the thick plugin in most environments, but if you wish to run the thin plugin, or are in a resource-constrained environment, you may do so with:
+
+```
+cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+```
+
+## Additional Installation Options
+
+In addition to the [quick-start guide](docs/quickstart.md), you may:
 
-- Install via daemonset using the quick-start guide, above.
 - Download binaries from [release page](https://github.com/k8snetworkplumbingwg/multus-cni/releases)
-- By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
+- By Docker image from [GitHub Container Registry](https://github.com/orgs/k8snetworkplumbingwg/packages/container/package/multus-cni)
 - Or, roll-your-own and build from source
   - See [Development](docs/development.md)
 
 ## Comprehensive Documentation
 
 - [How to use](docs/how-to-use.md)
+- [Quick Start Guide](docs/quickstart.md)
 - [Configuration](docs/configuration.md)
-- [Development](docs/development.md)
+- [Development and Support Information](docs/development.md)
+- [Thick Plugin](docs/thick-plugin.md)
 
 ## Contact Us
 
-For any questions about Multus CNI, feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/), or open up a GitHub issue. Request an invite to NPWG slack [here](https://intel-corp.herokuapp.com/).
+For any questions about Multus CNI, open up a GitHub issue or feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/).

cmd/config-generation/kubeconfig.go

@@ -1,148 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// this generates kubeconfig file for multus based on service account
-package main
-
-import (
-	"encoding/base64"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"strings"
-)
-
-const userRWPermission = 0600
-
-const (
-	cniConfigDirVarName   = "cni-config-dir"
-	k8sCAFilePathVarName  = "kube-ca-file"
-	k8sServiceHostVarName = "k8s-service-host"
-	k8sServicePortVarName = "k8s-service-port"
-	serviceAccountPath    = "/var/run/secrets/kubernetes.io/serviceaccount"
-	skipTLSVerifyVarName  = "skip-tls-verify"
-)
-
-const (
-	defaultCniConfigDir   = "/host/etc/cni/net.d"
-	defaultK8sCAFilePath  = ""
-	defaultK8sServiceHost = ""
-	defaultK8sServicePort = 0
-	defaultSkipTLSValue   = false
-)
-
-func main() {
-	k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
-	k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
-	skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
-	kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
-	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
-	flag.Parse()
-
-	if *k8sServiceHost == defaultK8sServiceHost {
-		logInvalidArg("must provide the k8s service cluster port")
-	}
-	if *k8sServicePort == defaultK8sServicePort {
-		logInvalidArg("must provide the k8s service cluster port")
-	}
-	if *kubeCAFilePath == defaultK8sServiceHost {
-		*kubeCAFilePath = serviceAccountPath + "/ca.crt"
-	}
-
-	tlsCfg := "insecure-skip-tls-verify: true"
-	if !*skipTLSVerify {
-		kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
-		if err != nil {
-			logError("failed grabbing CA file: %w", err)
-		}
-		tlsCfg = "certificate-authority-data: " + kubeCAFileContents
-	}
-
-	multusConfigDir := *cniConfigDir + "/multus.d/"
-	if err := prepareCNIConfigDir(multusConfigDir); err != nil {
-		logError("failed to create CNI config dir: %w", err)
-	}
-	kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
-	serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
-	if err != nil {
-		logError("failed grabbing k8s token: %w", err)
-	}
-	if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
-		logError("failed generating kubeconfig: %w", err)
-	}
-}
-
-func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
-	data, err := ioutil.ReadFile(pathCAFile)
-	if err != nil {
-		return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
-	}
-	return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
-}
-
-func k8sKubeConfigToken(tokenPath string) (string, error) {
-	data, err := ioutil.ReadFile(tokenPath)
-	if err != nil {
-		return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
-	}
-	return string(data), nil
-}
-
-func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
-	kubeConfigTemplate := `
-# Kubeconfig file for Multus CNI plugin.
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: %s://[%s]:%d
-    %s
-users:
-- name: multus
-  user:
-    token: "%s"
-contexts:
-- name: multus-context
-  context:
-    cluster: local
-    user: multus
-current-context: multus-context
-`
-	kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
-	logInfo("Generated KubeConfig saved to %s: \n%s", outputPath, kubeconfig)
-	return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
-}
-
-func prepareCNIConfigDir(cniConfigDirPath string) error {
-	return os.MkdirAll(cniConfigDirPath, userRWPermission)
-}
-
-func logInvalidArg(format string, values ...interface{}) {
-	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
-	flag.PrintDefaults()
-	os.Exit(1)
-}
-
-func logError(format string, values ...interface{}) {
-	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
-	os.Exit(1)
-}
-
-func logInfo(format string, values ...interface{}) {
-	log.Printf("INFO: %s", fmt.Sprintf(format, values...))
-}

cmd/controller/main.go

@@ -1,253 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// this is daemonized entrypoint process. which watches master config
-// and generate multus CNI config
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/config"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
-)
-
-const (
-	multusPluginName     = "multus"
-	multusConfigFileName = "00-multus.conf"
-)
-
-const (
-	defaultCniConfigDir                 = "/etc/cni/net.d"
-	defaultMultusAdditionalBinDir       = ""
-	defaultMultusCNIVersion             = ""
-	defaultMultusConfigFile             = "auto"
-	defaultMultusGlobalNamespaces       = ""
-	defaultMultusKubeconfigPath         = "/etc/cni/net.d/multus.d/multus.kubeconfig"
-	defaultMultusLogFile                = ""
-	defaultMultusLogMaxSize             = 100 // megabytes
-	defaultMultusLogMaxAge              = 5   // days
-	defaultMultusLogMaxBackups          = 5
-	defaultMultusLogCompress            = true
-	defaultMultusLogLevel               = ""
-	defaultMultusLogToStdErr            = false
-	defaultMultusMasterCNIFile          = ""
-	defaultMultusNamespaceIsolation     = false
-	defaultMultusReadinessIndicatorFile = ""
-)
-
-const (
-	cniConfigDirVarName           = "cni-config-dir"
-	multusAdditionalBinDirVarName = "additional-bin-dir"
-	multusAutoconfigDirVarName    = "multus-autoconfig-dir"
-	multusCNIVersion              = "cni-version"
-	multusConfigFileVarName       = "multus-conf-file"
-	multusGlobalNamespaces        = "global-namespaces"
-	multusLogFile                 = "multus-log-file"
-	multusLogMaxSize              = "multus-log-max-size"
-	multusLogMaxAge               = "multus-log-max-age"
-	multusLogMaxBackups           = "multus-log-max-backups"
-	multusLogCompress             = "multus-log-compress"
-	multusLogLevel                = "multus-log-level"
-	multusLogToStdErr             = "multus-log-to-stderr"
-	multusKubeconfigPath          = "multus-kubeconfig-file-host"
-	multusMasterCNIFileVarName    = "multus-master-cni-file"
-	multusNamespaceIsolation      = "namespace-isolation"
-	multusReadinessIndicatorFile  = "readiness-indicator-file"
-)
-
-func main() {
-	versionOpt := false
-	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
-	multusConfigFile := flag.String(multusConfigFileVarName, defaultMultusConfigFile, "The multus configuration file to use. By default, a new configuration is generated.")
-	multusMasterCni := flag.String(multusMasterCNIFileVarName, defaultMultusMasterCNIFile, "The relative name of the configuration file of the cluster primary CNI.")
-	multusAutoconfigDir := flag.String(multusAutoconfigDirVarName, *cniConfigDir, "The directory path for the generated multus configuration.")
-	namespaceIsolation := flag.Bool(multusNamespaceIsolation, defaultMultusNamespaceIsolation, "If the network resources are only available within their defined namespaces.")
-	globalNamespaces := flag.String(multusGlobalNamespaces, defaultMultusGlobalNamespaces, "Comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled.")
-	logToStdErr := flag.Bool(multusLogToStdErr, defaultMultusLogToStdErr, "If the multus logs are also to be echoed to stderr.")
-	logLevel := flag.String(multusLogLevel, defaultMultusLogLevel, "One of: debug/verbose/error/panic. Used only with --multus-conf-file=auto.")
-	logFile := flag.String(multusLogFile, defaultMultusLogFile, "Path where to multus will log. Used only with --multus-conf-file=auto.")
-	logMaxSize := flag.Int(multusLogMaxSize, defaultMultusLogMaxSize, "the maximum size in megabytes of the log file before it gets rotated")
-	logMaxAge := flag.Int(multusLogMaxAge, defaultMultusLogMaxAge, "the maximum number of days to retain old log files in their filename")
-	logMaxBackups := flag.Int(multusLogMaxBackups, defaultMultusLogMaxBackups, "the maximum number of old log files to retain")
-	logCompress := flag.Bool(multusLogCompress, defaultMultusLogCompress, "compress determines if the rotated log files should be compressed using gzip")
-	cniVersion := flag.String(multusCNIVersion, defaultMultusCNIVersion, "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
-	additionalBinDir := flag.String(multusAdditionalBinDirVarName, defaultMultusAdditionalBinDir, "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
-	readinessIndicator := flag.String(multusReadinessIndicatorFile, defaultMultusReadinessIndicatorFile, "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
-	multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
-	overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
-	flag.BoolVar(&versionOpt, "version", false, "Show application version")
-	flag.BoolVar(&versionOpt, "v", false, "Show application version")
-	flag.Parse()
-	if versionOpt == true {
-		fmt.Printf("%s\n", multus.PrintVersionString())
-		return
-	}
-
-	if *logToStdErr {
-		logging.SetLogStderr(*logToStdErr)
-	}
-	if *logFile != defaultMultusLogFile {
-		logging.SetLogFile(*logFile)
-	}
-	if *logLevel != defaultMultusLogLevel {
-		logging.SetLogLevel(*logLevel)
-	}
-
-	if *multusConfigFile == defaultMultusConfigFile {
-		if *cniVersion == defaultMultusCNIVersion {
-			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
-		}
-
-		var configurationOptions []config.Option
-		if *namespaceIsolation {
-			configurationOptions = append(
-				configurationOptions, config.WithNamespaceIsolation())
-		}
-
-		if *globalNamespaces != defaultMultusGlobalNamespaces {
-			configurationOptions = append(
-				configurationOptions, config.WithGlobalNamespaces(*globalNamespaces))
-		}
-
-		if *logToStdErr != defaultMultusLogToStdErr {
-			configurationOptions = append(
-				configurationOptions, config.WithLogToStdErr())
-		}
-
-		if *logLevel != defaultMultusLogLevel {
-			configurationOptions = append(
-				configurationOptions, config.WithLogLevel(*logLevel))
-		}
-
-		if *logFile != defaultMultusLogFile {
-			configurationOptions = append(
-				configurationOptions, config.WithLogFile(*logFile))
-		}
-
-		if *additionalBinDir != defaultMultusAdditionalBinDir {
-			configurationOptions = append(
-				configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
-		}
-
-		if *readinessIndicator != defaultMultusReadinessIndicatorFile {
-			configurationOptions = append(
-				configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
-		}
-
-		// logOptions
-
-		var logOptionFuncs []config.LogOptionFunc
-		if *logMaxAge != defaultMultusLogMaxAge {
-			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxAge(logMaxAge))
-		}
-		if *logMaxSize != defaultMultusLogMaxSize {
-			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxSize(logMaxSize))
-		}
-		if *logMaxBackups != defaultMultusLogMaxBackups {
-			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxBackups(logMaxBackups))
-		}
-		if *logCompress != defaultMultusLogCompress {
-			logOptionFuncs = append(logOptionFuncs, config.WithLogCompress(logCompress))
-		}
-
-		if len(logOptionFuncs) > 0 {
-			logOptions := &config.LogOptions{}
-			config.MutateLogOptions(logOptions, logOptionFuncs...)
-			configurationOptions = append(configurationOptions, config.WithLogOptions(logOptions))
-		}
-
-		multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
-		if err != nil {
-			_ = logging.Errorf("Failed to create multus config: %v", err)
-			os.Exit(3)
-		}
-
-		var configManager *config.Manager
-		if *multusMasterCni == "" {
-			configManager, err = config.NewManager(*multusConfig, *multusAutoconfigDir)
-		} else {
-			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
-				*multusConfig, *multusAutoconfigDir, *multusMasterCni)
-		}
-		if err != nil {
-			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
-			os.Exit(2)
-		}
-
-		if *overrideNetworkName {
-			if err := configManager.OverrideNetworkName(); err != nil {
-				_ = logging.Errorf("could not override the network name: %v", err)
-			}
-		}
-
-		generatedMultusConfig, err := configManager.GenerateConfig()
-		if err != nil {
-			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
-		}
-		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
-
-		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
-			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
-		}
-
-		configWatcherDoneChannel := make(chan struct{})
-		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
-			defer func() {
-				stopChannel <- struct{}{}
-			}()
-			if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
-				_ = logging.Errorf("error watching file: %v", err)
-			}
-		}(make(chan struct{}), configWatcherDoneChannel)
-
-		<-configWatcherDoneChannel
-	} else {
-		if err := copyUserProvidedConfig(*multusConfigFile, *cniConfigDir); err != nil {
-			logging.Errorf("failed to copy the user provided configuration %s: %v", *multusConfigFile, err)
-		}
-	}
-}
-
-func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
-	srcFile, err := os.Open(multusConfigPath)
-	if err != nil {
-		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
-	}
-
-	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
-	dstFile, err := os.Create(dstFileName)
-	if err != nil {
-		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
-	}
-	nBytes, err := io.Copy(srcFile, dstFile)
-	if err != nil {
-		return fmt.Errorf("error copying file: %w", err)
-	}
-	srcFileInfo, err := srcFile.Stat()
-	if err != nil {
-		return fmt.Errorf("failed to stat the file: %w", err)
-	} else if nBytes != srcFileInfo.Size() {
-		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
-	}
-	return nil
-}

cmd/install_multus/main.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a install tool for multus plugins
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+)
+
+func main() {
+	typeFlag := pflag.StringP("type", "t", "", "specify installer type (thick/thin)")
+	destDir := pflag.StringP("dest-dir", "d", "/host/opt/cni/bin", "destination directory")
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	multusFileName := ""
+	switch *typeFlag {
+	case "thick":
+		multusFileName = "multus-shim"
+	case "thin":
+		multusFileName = "multus"
+	default:
+		fmt.Fprintf(os.Stderr, "--type is missing or --type has invalid value\n")
+		os.Exit(1)
+	}
+
+	err := cmdutils.CopyFileAtomic(fmt.Sprintf("/usr/src/multus-cni/bin/%s", multusFileName), *destDir, fmt.Sprintf("%s.temp", multusFileName), multusFileName)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("multus %s copy succeeded!\n", multusFileName)
+}

cmd/multus-daemon/main.go

@@ -0,0 +1,217 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary works as a server that receives requests from multus-shim
+// CNI plugin and creates network interface for kubernets pods.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+	srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func main() {
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// keep in command line option
+	version := flag.Bool("version", false, "Show version")
+
+	configFilePath := flag.String("config", srv.DefaultMultusDaemonConfigFile, "Specify the path to the multus-daemon configuration")
+
+	flag.Parse()
+
+	if *version {
+		fmt.Printf("multus-daemon: %s\n", multus.PrintVersionString())
+		os.Exit(4)
+	}
+
+	configWatcherStopChannel := make(chan struct{})
+	configWatcherDoneChannel := make(chan struct{})
+	serverStopChannel := make(chan struct{})
+	serverDoneChannel := make(chan struct{})
+
+	daemonConf, err := cniServerConfig(*configFilePath)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	if err := startMultusDaemon(daemonConf, serverStopChannel, serverDoneChannel); err != nil {
+		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
+		os.Exit(3)
+	}
+
+	multusConf, err := config.ParseMultusConfig(*configFilePath)
+	if err != nil {
+		logging.Panicf("startMultusDaemon failed to load the multus configuration: %v", err)
+		os.Exit(1)
+	}
+
+	// Generate multus CNI config from current CNI config
+	if multusConf.MultusConfigFile == "auto" {
+		if multusConf.CNIVersion == "" {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		multusConf.SocketDir = daemonConf.SocketDir
+
+		var configManager *config.Manager
+		if multusConf.MultusMasterCni == "" {
+			configManager, err = config.NewManager(*multusConf, multusConf.MultusAutoconfigDir, multusConf.ForceCNIVersion)
+		} else {
+			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
+				*multusConf, multusConf.MultusAutoconfigDir, multusConf.MultusMasterCni, multusConf.ForceCNIVersion)
+		}
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+
+		if multusConf.OverrideNetworkName {
+			if err := configManager.OverrideNetworkName(); err != nil {
+				_ = logging.Errorf("could not override the network name: %v", err)
+			}
+		}
+
+		generatedMultusConfig, err := configManager.GenerateConfig()
+		if err != nil {
+			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
+		}
+		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
+			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+		}
+
+		go func(stopChannel chan<- struct{}, doneChannel chan<- struct{}) {
+			if err := configManager.MonitorPluginConfiguration(configWatcherStopChannel, doneChannel); err != nil {
+				_ = logging.Errorf("error watching file: %v", err)
+			}
+		}(configWatcherStopChannel, configWatcherDoneChannel)
+
+		<-configWatcherDoneChannel
+	} else {
+		if err := copyUserProvidedConfig(multusConf.MultusConfigFile, multusConf.CniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", multusConf.MultusConfigFile, err)
+		}
+	}
+
+	serverDone := false
+	configWatcherDone := false
+	for {
+		select {
+		case <-configWatcherDoneChannel:
+			logging.Verbosef("ConfigWatcher done")
+			configWatcherDone = true
+		case <-serverDoneChannel:
+			logging.Verbosef("multus-server done.")
+			serverDone = true
+		}
+
+		if serverDone && configWatcherDone {
+			return
+		}
+	}
+	// never reached
+}
+
+func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}, done chan struct{}) error {
+	if user, err := user.Current(); err != nil || user.Uid != "0" {
+		return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
+	}
+
+	if err := srv.FilesystemPreRequirements(daemonConfig.SocketDir); err != nil {
+		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
+	}
+
+	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents)
+	if err != nil {
+		return fmt.Errorf("failed to create the server: %v", err)
+	}
+
+	if daemonConfig.MetricsPort != nil {
+		go utilwait.Until(func() {
+			http.Handle("/metrics", promhttp.Handler())
+			logging.Debugf("metrics port: %d", *daemonConfig.MetricsPort)
+			logging.Debugf("metrics: %s", http.ListenAndServe(fmt.Sprintf(":%d", *daemonConfig.MetricsPort), nil))
+		}, 0, stopCh)
+	}
+
+	l, err := srv.GetListener(api.SocketPath(daemonConfig.SocketDir))
+	if err != nil {
+		return fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(daemonConfig.SocketDir), err)
+	}
+
+	server.SetKeepAlivesEnabled(false)
+	go func() {
+		utilwait.Until(func() {
+			logging.Debugf("open for business")
+			if err := server.Serve(l); err != nil {
+				utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
+			}
+		}, 0, stopCh)
+		server.Shutdown(context.TODO())
+		close(done)
+	}()
+
+	return nil
+}
+
+func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
+	configFileContents, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	return srv.LoadDaemonNetConf(configFileContents)
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	srcFile, err := os.Open(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(dstFile, srcFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/multus-shim/main.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept referred from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
+)
+
+func main() {
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+
+	flag.Parse()
+	if versionOpt {
+		fmt.Printf("multus-shim: %s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			return api.CmdAdd(args)
+		},
+		func(args *skel.CmdArgs) error {
+			return api.CmdCheck(args)
+		},
+		func(args *skel.CmdArgs) error {
+			return api.CmdDel(args)
+		},
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

cmd/multus/main.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2016 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,7 +25,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	cniversion "github.com/containernetworking/cni/pkg/version"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
 )
 
 func main() {
@@ -37,8 +38,8 @@ func main() {
 	flag.BoolVar(&versionOpt, "version", false, "Show application version")
 	flag.BoolVar(&versionOpt, "v", false, "Show application version")
 	flag.Parse()
-	if versionOpt == true {
-		fmt.Printf("%s\n", multus.PrintVersionString())
+	if versionOpt {
+		fmt.Printf("multus: %s\n", multus.PrintVersionString())
 		return
 	}
 

cmd/thin_entrypoint/main.go

@@ -0,0 +1,585 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a entrypoint for thin (stand-alone) images.
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	b64 "encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/containernetworking/cni/libcni"
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+)
+
+// Options stores command line options
+type Options struct {
+	CNIBinDir                string
+	CNIConfDir               string
+	CNIVersion               string
+	MultusConfFile           string
+	MultusBinFile            string // may be hidden or remove?
+	SkipMultusBinaryCopy     bool
+	MultusKubeConfigFileHost string
+	MultusMasterCNIFileName  string
+	NamespaceIsolation       bool
+	GlobalNamespaces         string
+	MultusAutoconfigDir      string
+	MultusLogToStderr        bool
+	MultusLogLevel           string
+	MultusLogFile            string
+	OverrideNetworkName      bool
+	CleanupConfigOnExit      bool
+	RenameConfFile           bool
+	ReadinessIndicatorFile   string
+	AdditionalBinDir         string
+	ForceCNIVersion          bool
+	SkipTLSVerify            bool
+}
+
+const (
+	serviceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+	serviceAccountCAFile    = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+)
+
+func (o *Options) addFlags() {
+	pflag.ErrHelp = nil // suppress error message for help
+	fs := pflag.CommandLine
+	fs.StringVar(&o.CNIBinDir, "cni-bin-dir", "/host/opt/cni/bin", "CNI binary directory")
+	fs.StringVar(&o.CNIConfDir, "cni-conf-dir", "/host/etc/cni/net.d", "CNI config directory")
+	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
+	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
+	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
+	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")
+
+	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.MultusMasterCNIFileName, "multus-master-cni-file-name", "", "master CNI file in multus-autoconfig-dir")
+	fs.BoolVar(&o.NamespaceIsolation, "namespace-isolation", false, "namespace isolation")
+	fs.StringVar(&o.GlobalNamespaces, "global-namespaces", "", "global namespaces, comma separated (used only with --namespace-isolation=true)")
+	fs.StringVar(&o.MultusAutoconfigDir, "multus-autoconfig-dir", "/host/etc/cni/net.d", "multus autoconfig dir (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.MultusLogToStderr, "multus-log-to-stderr", true, "log to stderr")
+	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
+	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
+	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.SkipTLSVerify, "skip-tls-verify", false, "skip TLS verify")
+	fs.BoolVar(&o.ForceCNIVersion, "force-cni-version", false, "force cni version to '--cni-version' (only for e2e-kind testing)")
+	fs.MarkHidden("force-cni-version")
+	fs.MarkHidden("skip-tls-verify")
+}
+
+func (o *Options) verifyFileExists() error {
+	// CNIConfDir
+	if _, err := os.Stat(o.CNIConfDir); err != nil {
+		return fmt.Errorf("cni-conf-dir is not found: %v", err)
+	}
+
+	// CNIBinDir
+	if _, err := os.Stat(o.CNIBinDir); err != nil {
+		return fmt.Errorf("cni-bin-dir is not found: %v", err)
+	}
+
+	// MultusBinFile
+	if _, err := os.Stat(o.MultusBinFile); err != nil {
+		return fmt.Errorf("multus-bin-file is not found: %v", err)
+	}
+
+	if o.MultusConfFile != "auto" {
+		// MultusConfFile
+		if _, err := os.Stat(o.MultusConfFile); err != nil {
+			return fmt.Errorf("multus-conf-file is not found: %v", err)
+		}
+	}
+	return nil
+}
+
+const kubeConfigTemplate = `# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: {{ .KubeConfigHost }}
+    {{ .KubeServerTLS }}
+users:
+- name: multus
+  user:
+    token: "{{ .KubeServiceAccountToken }}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+
+func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
+	// check file exists
+	if _, err := os.Stat(serviceAccountTokenFile); err != nil {
+		return nil, fmt.Errorf("service account token is not found: %v", err)
+	}
+	if _, err := os.Stat(serviceAccountCAFile); err != nil {
+		return nil, fmt.Errorf("service account ca is not found: %v", err)
+	}
+
+	// create multus.d directory
+	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
+		return nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+	}
+
+	// get Kubernetes service protocol/host/port
+	kubeProtocol := os.Getenv("KUBERNETES_SERVICE_PROTOCOL")
+	if kubeProtocol == "" {
+		kubeProtocol = "https"
+	}
+	kubeHost := os.Getenv("KUBERNETES_SERVICE_HOST")
+	kubePort := os.Getenv("KUBERNETES_SERVICE_PORT")
+
+	// check tlsConfig
+	tlsConfig := ""
+	if o.SkipTLSVerify {
+		tlsConfig = "insecure-skip-tls-verify: true"
+	} else {
+		// create tlsConfig by service account CA file
+		caFileByte, err := os.ReadFile(serviceAccountCAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read service account ca file: %v", err)
+		}
+		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
+		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
+	}
+
+	saTokenByte, err := os.ReadFile(serviceAccountTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read service account token file: %v", err)
+	}
+
+	// create kubeconfig by template and replace it by atomic
+	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
+	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
+	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
+	}
+
+	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("template parse error: %v", err)
+	}
+	templateData := map[string]string{
+		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
+		"KubeServerTLS":           tlsConfig,
+		"KubeServiceAccountToken": string(saTokenByte),
+	}
+
+	// Prepare
+	hash := sha256.New()
+	writer := io.MultiWriter(hash, fp)
+
+	// genearate kubeconfig from template
+	if err = templateKubeconfig.Execute(writer, templateData); err != nil {
+		return nil, fmt.Errorf("cannot create kubeconfig: %v", err)
+	}
+
+	if err := fp.Sync(); err != nil {
+		os.Remove(fp.Name())
+		return nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
+	}
+	if err := fp.Close(); err != nil {
+		os.Remove(fp.Name())
+		return nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
+	}
+
+	newFileHash := hash.Sum(nil)
+	if currentFileHash != nil && bytes.Compare(newFileHash, currentFileHash) == 0 {
+		fmt.Printf("kubeconfig is same, not copy\n")
+		os.Remove(fp.Name())
+		return currentFileHash, nil
+	}
+
+	// replace file with tempfile
+	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
+		return nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
+	}
+
+	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
+	return newFileHash, nil
+}
+
+const multusConflistTemplate = `{
+    "cniVersion": "{{ .CNIVersion }}",
+    "name": "{{ .MasterPluginNetworkName }}",
+    "plugins": [ {
+        "type": "multus",{{
+            .NestedCapabilities
+        }}{{
+            .NamespaceIsolationConfig
+        }}{{
+            .GlobalNamespacesConfig
+        }}{{
+            .LogToStderrConfig
+        }}{{
+            .LogLevelConfig
+        }}{{
+            .LogFileConfig
+        }}{{
+            .AdditionalBinDirConfig
+        }}{{
+            .ReadinessIndicatorFileConfig
+        }}
+        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
+        "delegates": [
+            {{ .MasterPluginJSON }}
+        ]
+    }]
+}
+`
+
+const multusConfTemplate = `{
+        "cniVersion": "{{ .CNIVersion }}",
+        "name": "{{ .MasterPluginNetworkName }}",
+        "type": "multus",{{
+            .NestedCapabilities
+        }}{{
+            .NamespaceIsolationConfig
+        }}{{
+            .GlobalNamespacesConfig
+        }}{{
+            .LogToStderrConfig
+        }}{{
+            .LogLevelConfig
+        }}{{
+            .LogFileConfig
+        }}{{
+            .AdditionalBinDirConfig
+        }}{{
+            .ReadinessIndicatorFileConfig
+        }}
+        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
+        "delegates": [
+                {{ .MasterPluginJSON }}
+        ]
+}
+`
+
+func (o *Options) createMultusConfig() (string, error) {
+	// find master file from MultusAutoconfigDir
+	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
+	if err != nil {
+		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
+	}
+
+	masterConfigPath := files[0]
+	masterConfigBytes, err := os.ReadFile(masterConfigPath)
+	if err != nil {
+		return "", fmt.Errorf("cannot read master CNI config file %q: %v", masterConfigPath, err)
+	}
+	masterConfig := map[string]interface{}{}
+	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
+		return "", fmt.Errorf("cannot read master CNI config json: %v", err)
+	}
+
+	// check CNIVersion
+	masterCNIVersionElem, ok := masterConfig["cniVersion"]
+	if !ok {
+		return "", fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
+	}
+
+	if o.ForceCNIVersion {
+		masterConfig["cniVersion"] = o.CNIVersion
+		fmt.Printf("force CNI version to %q\n", o.CNIVersion)
+	} else {
+		masterCNIVersion := masterCNIVersionElem.(string)
+		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
+			return "", fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
+		}
+		o.CNIVersion = masterCNIVersion
+	}
+	cniVersionConfig := o.CNIVersion
+
+	// check OverrideNetworkName (if true, get master plugin name, otherwise 'multus-cni-network'
+	masterPluginNetworkName := "multus-cni-network"
+	if o.OverrideNetworkName {
+		masterPluginNetworkElem, ok := masterConfig["name"]
+		if !ok {
+			return "", fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
+		}
+
+		masterPluginNetworkName = masterPluginNetworkElem.(string)
+		fmt.Printf("master plugin name is overrided to %q\n", masterPluginNetworkName)
+	}
+
+	// check capabilities (from master conf, top and 'plugins')
+	masterCapabilities := map[string]bool{}
+	_, isMasterConfList := masterConfig["plugins"]
+
+	if isMasterConfList {
+		masterPluginsElem, ok := masterConfig["plugins"]
+		if !ok {
+			return "", fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
+		}
+		masterPlugins := masterPluginsElem.([]interface{})
+		for _, v := range masterPlugins {
+			pluginFields := v.(map[string]interface{})
+			capabilitiesElem, ok := pluginFields["capabilities"]
+			if ok {
+				capabilities := capabilitiesElem.(map[string]interface{})
+				for k, v := range capabilities {
+					masterCapabilities[k] = v.(bool)
+				}
+			}
+		}
+		fmt.Printf("master capabilities is get from conflist\n")
+	} else {
+		masterCapabilitiesElem, ok := masterConfig["capabilities"]
+		if ok {
+			for k, v := range masterCapabilitiesElem.(map[string]interface{}) {
+				masterCapabilities[k] = v.(bool)
+			}
+		}
+		fmt.Printf("master capabilities is get from conffile\n")
+	}
+	nestedCapabilitiesConf := ""
+	if len(masterCapabilities) != 0 {
+		capabilitiesByte, err := json.Marshal(masterCapabilities)
+		if err != nil {
+			return "", fmt.Errorf("cannot get capabilities map: %v", err)
+		}
+		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
+	}
+
+	// check NamespaceIsolation
+	namespaceIsolationConfig := ""
+	if o.NamespaceIsolation {
+		namespaceIsolationConfig = "\n        \"namespaceIsolation\": true,"
+	}
+
+	// check GlobalNamespaces
+	globalNamespaceConfig := ""
+	if o.GlobalNamespaces != "" {
+		globalNamespaceConfig = fmt.Sprintf("\n        \"globalNamespaces\": %q,", o.GlobalNamespaces)
+	}
+
+	// check MultusLogToStderr
+	logToStderrConfig := ""
+	if !o.MultusLogToStderr {
+		logToStderrConfig = "\n        \"logToStderr\": false,"
+	}
+
+	// check MultusLogLevel (debug/error/panic/verbose) and reject others
+	logLevelConfig := ""
+	logLevelStr := strings.ToLower(o.MultusLogLevel)
+	switch logLevelStr {
+	case "debug", "error", "panic", "verbose":
+		logLevelConfig = fmt.Sprintf("\n        \"logLevel\": %q,", logLevelStr)
+	case "":
+		// no logLevel config, skipped
+	default:
+		return "", fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
+	}
+
+	// check MultusLogFile
+	logFileConfig := ""
+	if o.MultusLogFile != "" {
+		logFileConfig = fmt.Sprintf("\n        \"logFile\": %q,", o.MultusLogFile)
+	}
+
+	// check AdditionalBinDir
+	additionalBinDirConfig := ""
+	if o.AdditionalBinDir != "" {
+		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
+	}
+
+	// check ReadinessIndicatorFile
+	readinessIndicatorFileConfig := ""
+	if o.ReadinessIndicatorFile != "" {
+		readinessIndicatorFileConfig = fmt.Sprintf("\n        \"readinessindicatorfile\": %q,", o.ReadinessIndicatorFile)
+	}
+
+	// fill .MasterPluginJSON
+	masterPluginByte, err := json.Marshal(masterConfig)
+	if err != nil {
+		return "", fmt.Errorf("cannot encode master CNI config: %v", err)
+	}
+
+	// generate multus config
+	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
+	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return "", fmt.Errorf("cannot create multus cni temp file: %v", err)
+	}
+
+	// use conflist template if cniVersionConfig == "1.0.0"
+	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
+	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
+	if err != nil {
+		return "", fmt.Errorf("template parse error: %v", err)
+	}
+
+	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
+		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
+		if err != nil {
+			return "", fmt.Errorf("template parse error: %v", err)
+		}
+	}
+
+	templateData := map[string]string{
+		"CNIVersion":                   cniVersionConfig,
+		"MasterPluginNetworkName":      masterPluginNetworkName,
+		"NestedCapabilities":           nestedCapabilitiesConf,
+		"NamespaceIsolationConfig":     namespaceIsolationConfig,
+		"GlobalNamespacesConfig":       globalNamespaceConfig,
+		"LogToStderrConfig":            logToStderrConfig,
+		"LogLevelConfig":               logLevelConfig,
+		"LogFileConfig":                logFileConfig,
+		"AdditionalBinDirConfig":       additionalBinDirConfig,
+		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
+		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
+		"MasterPluginJSON":             string(masterPluginByte),
+	}
+	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
+		return "", fmt.Errorf("cannot create multus cni config: %v", err)
+	}
+
+	if err := fp.Sync(); err != nil {
+		os.Remove(tempFileName)
+		return "", fmt.Errorf("cannot flush multus cni config: %v", err)
+	}
+	if err := fp.Close(); err != nil {
+		os.Remove(tempFileName)
+		return "", fmt.Errorf("cannot close multus cni config: %v", err)
+	}
+
+	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
+		return "", fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
+	}
+
+	if o.RenameConfFile {
+		//masterConfigPath
+		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
+		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
+			return "", fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
+		}
+		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
+	}
+
+	return masterConfigPath, nil
+}
+
+func main() {
+	opt := Options{}
+	opt.addFlags()
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	err := opt.verifyFileExists()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		return
+	}
+
+	// copy multus binary
+	if !opt.SkipMultusBinaryCopy {
+		// Copy
+		if err = cmdutils.CopyFileAtomic(opt.MultusBinFile, opt.CNIBinDir, "_multus", "multus"); err != nil {
+			fmt.Fprintf(os.Stderr, "failed at multus copy: %v\n", err)
+			return
+		}
+	}
+
+	var kubeConfigHash []byte
+	var masterConfigFilePath string
+	// copy user specified multus conf to CNI conf directory
+	if opt.MultusConfFile != "auto" {
+		confFileName := filepath.Base(opt.MultusConfFile)
+		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
+		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
+			fmt.Fprintf(os.Stderr, "failed at copy multus conf file: %v\n", err)
+			return
+		}
+		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
+	} else { // auto generate multus config
+		kubeConfigHash, err = opt.createKubeConfig(nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
+			return
+		}
+		fmt.Printf("kubeconfig file is created.\n")
+		masterConfigFilePath, err = opt.createMultusConfig()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+			return
+		}
+		fmt.Printf("multus config file is created.\n")
+	}
+
+	if opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" {
+		fmt.Printf("Entering watch loop...\n")
+		for {
+			// Check kubeconfig and update if different (i.e. service account updated)
+			kubeConfigHash, err = opt.createKubeConfig(kubeConfigHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
+				return
+			}
+
+			// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
+			_, err = os.Stat(masterConfigFilePath)
+
+			// if masterConfigFilePath is no longer exists
+			if os.IsNotExist(err) {
+				fmt.Printf("Master plugin @ %q has been deleted. Allowing 45 seconds for its restoration...\n", masterConfigFilePath)
+				time.Sleep(10 * time.Second)
+
+				for range time.Tick(1 * time.Second) {
+					_, err = os.Stat(masterConfigFilePath)
+					if !os.IsNotExist(err) {
+						fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
+						break
+					}
+				}
+			}
+			masterConfigFilePath, err = opt.createMultusConfig()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+				return
+			}
+		}
+	} else {
+		// sleep infinitely
+		for {
+			time.Sleep(time.Duration(1<<63 - 1))
+		}
+	}
+}

cmd/thin_entrypoint/main_test.go

@@ -0,0 +1,402 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestThinEntrypoint(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "thin_entrypoint")
+}
+
+var _ = Describe("thin entrypoint testing", func() {
+	It("always pass just example", func() {
+		a := 10
+		Expect(a).To(Equal(10))
+	})
+
+	It("Run verifyFileExists() with expected environment, autoconfig", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
+		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
+		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
+		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
+
+		// CNIConfDir
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// CNIBinDir
+		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
+
+		// MultusBinFile
+		Expect(os.WriteFile(multusBinFile, nil, 0744)).To(Succeed())
+
+		// MultusConfFile
+		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
+
+		err = (&Options{
+			CNIConfDir:     cniConfDir,
+			CNIBinDir:      cniBinDir,
+			MultusBinFile:  multusBinFile,
+			MultusConfFile: multusConfFile,
+		}).verifyFileExists()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run verifyFileExists() with invalid environmentMultusConfFile", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
+		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
+		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
+		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
+
+		// CNIConfDir
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// CNIBinDir
+		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
+
+		// MultusConfFile
+		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
+
+		err = (&Options{
+			CNIConfDir:     cniConfDir,
+			CNIBinDir:      cniBinDir,
+			MultusBinFile:  multusBinFile,
+			MultusConfFile: multusConfFile,
+		}).verifyFileExists()
+		Expect(err).To(HaveOccurred())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), default, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), capabilities, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"capabilities": { "bandwidth": true },
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "capabilities": {"bandwidth":true},
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"capabilities":{"bandwidth":true},"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), with options, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		err = os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+			NamespaceIsolation:       true,
+			GlobalNamespaces:         "foobar,barfoo",
+			MultusLogToStderr:        true,
+			MultusLogLevel:           "DEBUG",
+			MultusLogFile:            "/tmp/foobar.log",
+			AdditionalBinDir:         "/tmp/add_bin_dir",
+			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "namespaceIsolation": true,
+        "globalNamespaces": "foobar,barfoo",
+        "logLevel": "debug",
+        "logFile": "/tmp/foobar.log",
+        "binDir": "/tmp/add_bin_dir",
+        "readinessindicatorfile": "/var/lib/foobar_indicator",
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), default, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), capabilities, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"capabilities": { "bandwidth": true },
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "capabilities": {"bandwidth":true},
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"capabilities":{"bandwidth":true},"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), with options, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+			NamespaceIsolation:       true,
+			GlobalNamespaces:         "foobar,barfoo",
+			MultusLogToStderr:        true,
+			MultusLogLevel:           "DEBUG",
+			MultusLogFile:            "/tmp/foobar.log",
+			AdditionalBinDir:         "/tmp/add_bin_dir",
+			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
+		}).createMultusConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "namespaceIsolation": true,
+        "globalNamespaces": "foobar,barfoo",
+        "logLevel": "debug",
+        "logFile": "/tmp/foobar.log",
+        "binDir": "/tmp/add_bin_dir",
+        "readinessindicatorfile": "/var/lib/foobar_indicator",
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+})

deployments/deprecated/multus-daemonset-crio-pre1.16.yml

@@ -1,263 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
-            Working Group to express the intent for attaching pods to one or more logical or physical
-            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
-          type: object
-          properties:
-            spec:
-              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
-              type: object
-              properties:
-                config:
-                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:v3.6
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--override-network-name=true"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-          capabilities:
-            add: ["SYS_ADMIN"]
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--override-network-name=true"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

deployments/deprecated/multus-daemonset-gke-pre-1.16.yml

@@ -1,232 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  version: v1
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-      - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-              type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-  - kind: ServiceAccount
-    name: multus
-    namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          image: nfvpe/multus:v3.6
-          command: ["/entrypoint.sh"]
-          args:
-            - "--multus-conf-file=auto"
-            - "--cni-bin-dir=/host/home/kubernetes/bin"
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "50Mi"
-            limits:
-              cpu: "100m"
-              memory: "50Mi"
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /host/home/kubernetes/bin
-            - name: multus-cfg
-              mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /home/kubernetes/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-              - key: cni-conf.json
-                path: 70-multus.conf
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: ppc64le
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          # ppc64le support requires multus:latest for now. support 3.3 or later.
-          image: nfvpe/multus:latest-ppc64le
-          command: ["/entrypoint.sh"]
-          args:
-            - "--multus-conf-file=auto"
-            - "--cni-bin-dir=/host/home/kubernetes/bin"
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "90Mi"
-            limits:
-              cpu: "100m"
-              memory: "90Mi"
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /host/home/kubernetes/bin
-            - name: multus-cfg
-              mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /home/kubernetes/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-              - key: cni-conf.json
-                path: 70-multus.conf

deployments/deprecated/multus-daemonset-pre-1.16.yml

@@ -1,249 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
-            Working Group to express the intent for attaching pods to one or more logical or physical
-            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
-          type: object
-          properties:
-            spec:
-              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
-              type: object
-              properties:
-                config:
-                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: nfvpe/multus:v3.6
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # ppc64le support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

deployments/multus-daemonset-crio.yml

@@ -1,3 +1,11 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -178,7 +186,6 @@ spec:
         - "--cni-version=0.3.1"
         - "--cni-bin-dir=/host/usr/libexec/cni"
         - "--multus-conf-file=auto"
-        - "--restart-crio=true"
         resources:
           requests:
             cpu: "100m"

deployments/multus-daemonset-gke-1.16.yml

@@ -1,181 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-      - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              properties:
-                config:
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-  - kind: ServiceAccount
-    name: multus
-    namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-        - operator: Exists
-          effect: NoExecute
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
-          command: ["/entrypoint.sh"]
-          args:
-            - "--multus-conf-file=auto"
-            - "--cni-version=0.3.1"
-            - "--cni-bin-dir=/host/home/kubernetes/bin"
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "50Mi"
-            limits:
-              cpu: "100m"
-              memory: "50Mi"
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /host/home/kubernetes/bin
-            - name: multus-cfg
-              mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /home/kubernetes/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-              - key: cni-conf.json
-                path: 70-multus.conf

deployments/multus-daemonset-thick.yml

@@ -1,3 +1,11 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -91,6 +99,27 @@ metadata:
   name: multus
   namespace: kube-system
 ---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "chrootDir": "/hostroot",
+        "confDir": "/host/etc/cni/net.d",
+        "logLevel": "verbose",
+        "socketDir": "/host/run/multus/",
+        "cniVersion": "0.3.1",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "multusAutoconfigDir": "/host/etc/cni/net.d"
+    }
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -114,6 +143,7 @@ spec:
         name: multus
     spec:
       hostNetwork: true
+      hostPID: true
       tolerations:
         - operator: Exists
           effect: NoSchedule
@@ -122,14 +152,8 @@ spec:
       serviceAccountName: multus
       containers:
         - name: kube-multus
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
           command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
-          args:
-            - "-cni-version=0.3.1"
-            - "-cni-config-dir=/host/etc/cni/net.d"
-            - "-multus-autoconfig-dir=/host/etc/cni/net.d"
-            - "-multus-log-to-stderr=true"
-            - "-multus-log-level=verbose"
           resources:
             requests:
               cpu: "100m"
@@ -142,15 +166,30 @@ spec:
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
-            - name: cnibin
-              mountPath: /host/opt/cni/bin
+            - name: host-run
+              mountPath: /host/run
+            - name: host-var-lib-cni-multus
+              mountPath: /var/lib/cni/multus
+            - name: host-var-lib-kubelet
+              mountPath: /var/lib/kubelet
+            - name: host-run-k8s-cni-cncf-io
+              mountPath: /run/k8s.cni.cncf.io
+            - name: host-run-netns
+              mountPath: /run/netns
+              mountPropagation: HostToContainer
+            - name: multus-daemon-config
+              mountPath: /etc/cni/net.d/multus.d
+              readOnly: true
+            - name: hostroot
+              mountPath: /hostroot
+              mountPropagation: HostToContainer
       initContainers:
         - name: install-multus-binary
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
           command:
             - "cp"
-            - "/usr/src/multus-cni/bin/multus"
-            - "/host/opt/cni/bin/multus"
+            - "/usr/src/multus-cni/bin/multus-shim"
+            - "/host/opt/cni/bin/multus-shim"
           resources:
             requests:
               cpu: "10m"
@@ -161,23 +200,6 @@ spec:
             - name: cnibin
               mountPath: /host/opt/cni/bin
               mountPropagation: Bidirectional
-        - name: generate-kubeconfig
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:thick
-          command:
-            - "/usr/src/multus-cni/bin/generate-kubeconfig"
-          args:
-            - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
-            - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
-          resources:
-            requests:
-              cpu: "10m"
-              memory: "15Mi"
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-              mountPropagation: Bidirectional
       terminationGracePeriodSeconds: 10
       volumes:
         - name: cni
@@ -186,4 +208,27 @@ spec:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
-
+        - name: hostroot
+          hostPath:
+            path: /
+        - name: multus-daemon-config
+          configMap:
+            name: multus-daemon-config
+            items:
+            - key: daemon-config.json
+              path: daemon-config.json
+        - name: host-run
+          hostPath:
+            path: /run
+        - name: host-var-lib-cni-multus
+          hostPath:
+            path: /var/lib/cni/multus
+        - name: host-var-lib-kubelet
+          hostPath:
+            path: /var/lib/kubelet
+        - name: host-run-k8s-cni-cncf-io
+          hostPath:
+            path: /run/k8s.cni.cncf.io
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

deployments/multus-daemonset.yml

@@ -1,3 +1,11 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -171,11 +179,12 @@ spec:
       serviceAccountName: multus
       containers:
       - name: kube-multus
-        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
-        command: ["/entrypoint.sh"]
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
+        command: ["/thin_entrypoint"]
         args:
         - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
+        - "--multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "--cni-conf-dir=/host/etc/cni/net.d"
         resources:
           requests:
             cpu: "100m"
@@ -194,11 +203,11 @@ spec:
           mountPath: /tmp/multus-conf
       initContainers:
         - name: install-multus-binary
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
-          command:
-            - "cp"
-            - "/usr/src/multus-cni/bin/multus"
-            - "/host/opt/cni/bin/multus"
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
+          command: ["/install_multus"]
+          args:
+            - "--type"
+            - "thin"
           resources:
             requests:
               cpu: "10m"

docs/configuration.md

@@ -37,7 +37,8 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     }, {
         "type": "macvlan",
         ... (snip)
-    }]
+    }],
+    allowTryDeleteOnErr: false
 }
 ```
 
@@ -57,11 +58,12 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
 
 User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
 
-* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
-* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
+* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
 * `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
 * `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
 * `delegates` ([]map,required): number of delegate details in the Multus
+* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL message to next when some missing error. Defaults to false.
 
 ### Network selection flow of clusterNetwork/defaultNetworks
 
@@ -70,6 +72,7 @@ Multus will find network for clusterNetwork/defaultNetworks as following sequenc
 1. CRD object for given network name, in 'kube-system' namespace
 1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
 1. Directory for CNI json config file. Multus will find alphabetically first file for the network
+1. File path for CNI json confile file.
 1. Multus failed to find network. Multus raise error message
 
 ## Miscellaneous config

docs/development.md

@@ -1,4 +1,14 @@
-## Development Information
+## Development/Support Information
+
+## Which Kubernetes version is supported in multus?
+
+Currently multus team supports Kubernetes that Kubernetes community maintains.
+See [Version Skew Policy](https://kubernetes.io/releases/version-skew-policy/) for the details.
+
+## How to debug multus-cni thin image?
+
+Latest multus uses [distroless](https://github.com/GoogleContainerTools/distroless) container image for its base,
+hence there is no shell command. If you want to execute shell in multus pod, please use `-debug` image (e.g. ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-debug), which has shell.
 
 ## How to utilize multus-cni code as library?
 
@@ -6,10 +16,9 @@ Multus now uses [gopkg.in](http://gopkg.in/) to expose its code as library.
 You can use following command to import our code into your go code.
 
 ```
-go get gopkg.in/k8snetworkplumbingwg/multus-cni.v3
+go get gopkg.in/k8snetworkplumbingwg/multus-cni.v4
 ```
 
-
 ## How do I submit an issue?
 
 Use GitHub as normally, you'll be presented with an option to submit a issue or enhancement request.

docs/how-to-use.md

@@ -21,7 +21,11 @@ You may acquire the Multus binary via compilation (see the [developer guide](dev
 
 As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+    cat ./deployments/multus-daemonset.yml | kubectl apply -f -  # thin deployment
+
+or
+
+    cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f - # thick (client/server) deployment
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -39,7 +43,7 @@ cat >/etc/cni/net.d/00-multus.conf <<EOF
 {
   "name": "multus-cni-network",
   "type": "multus",
-  "readinessindicatorfile": "/var/run/flannel/subnet.env",
+  "readinessindicatorfile": "/run/flannel/subnet.env",
   "delegates": [
     {
       "NOTE1": "This is example, wrote your CNI config in delegates",
@@ -260,6 +264,7 @@ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
       ]
   }
 }
+EOF
 ```
 
 ### Run pod with network annotation
@@ -614,10 +619,6 @@ In some cases, the original CNI configuration that the Multus configuration was
 
     --cleanup-config-on-exit=true
 
-When using CRIO, you may need to restart CRIO to get the Multus configuration file to take -- this is rarely necessary.
-
-    --restart-crio=false
-
 Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
 
     --rename-conf-file=true

docs/quickstart.md

@@ -48,10 +48,19 @@ Firstly, clone this GitHub repository.
 git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
 ```
 
-We'll apply a YAML file with `kubectl` from this repo.
+We'll apply a YAML file with `kubectl` from this repo, which installs the Multus components.
+
+Recommended installation:
 
 ```
-cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
+cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
+```
+See the [thick plugin docs](./thick-plugin.md) for more information about this architecture.
+
+Alternatively, you may install the thin-plugin with:
+
+```
+cat ./deployments/multus-daemonset.yml | kubectl apply -f -
 ```
 
 ### What the Multus daemonset does

docs/thick-plugin.md

@@ -0,0 +1,113 @@
+# Multus Thick plugin
+
+Multus CNI can also be deployed using a thick plugin architecture, which is
+characterized by a client/server architecture.
+
+The client - which will be referred to as "shim" - is a binary executable
+located on the Kubernetes node's file-system that
+[speaks CNI](https://github.com/containernetworking/cni/blob/master/SPEC.md#section-2-execution-protocol):
+the runtime - Kubernetes - passes parameters to the plugin via environment
+variables and configuration - which is passed via stdin.
+The plugin returns a result on stdout on success, or an error on stderr if the
+operation fails. Configuration and results are a JSON encoded string.
+
+Once the shim is invoked by the runtime (Kubernetes) it will contact the
+multus-daemon (server) via a unix domain socket which is bind mounted to the
+host's file-system; the multus-daemon is the one that will do all the
+heavy-pulling: fetch the delegate CNI configuration from the corresponding
+`net-attach-def`, compute the `RuntimeConfig`, and finally, invoke the delegate.
+
+It will then return the result of the operation back to the client.
+
+Please refer to the diagram below for a visual representation of the flow
+described above:
+
+```
+┌─────────┐             ┌───────┐           ┌────────┐             ┌──────────┐
+│         │ cni ADD/DEL │       │ REST POST │        │ cni ADD/DEL │          │
+│ runtime ├────────────►│ shim  │===========│ daemon ├────────────►│ delegate │
+│         │<------------│       │           │        │<------------│          │
+└─────────┘             └───────┘           └────────┘             └──────────┘
+```
+
+## How to use it
+
+### Configure Deployment
+
+If your delegate CNI plugin requires some files which is in container host, please update
+update `deployments/multus-daemonset-thick.yml` to add directory into multus-daemon pod.
+For example, flannel requires `/run/flannel/subnet.env`, so you need to mount this directory
+into the multus-daemon pod.
+
+Required directory/files are different for each CNI plugin, so please refer your CNI plugin.
+
+### Deployment
+
+There is a dedicated multus daemonset specification for users wanting to use
+this thick plugin variant. This reference deployment spec of multus can be
+deployed by following these commands:
+
+```bash
+kubectl apply -f deployments/multus-daemonset-thick.yml
+```
+
+### Command line parameters
+
+The available command line parameters are:
+
+- `config`: Defaults to `"/etc/cni/net.d/multus.d/daemon-config.json"`
+- `version`: Prints the daemon config version and exits
+
+### Server / Daemon configuration
+
+The server configuration is encoded in JSON, and allows the following keys:
+
+- `"chrootDir"`: Specify the directory which points to host root from the pod. See 'Chroot configuration' section for the details.
+- `"socketDir"`: Specify the location where the unix domain socket used
+for client/server communication will be located. This is the location where the
+**Daemon** will read the configuration from. Defaults to `"/run/multus"`.
+- `"metricsPort"`: Metrics port (of multus' metric exporter); by default, no port
+is provided.
+- `"logFile"`: the path to where the daemon logs will be persisted.
+- `"logLevel"`: the logging level for the multus daemon logs.
+- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
+as well. By default, it is disabled.
+
+In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
+
+Below you can see an example of the daemon configuration:
+```json
+{
+        "chrootDir": "/hostroot",
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "verbose",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus/",
+        "cniVersion": "0.3.1",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "multusAutoconfigDir": "/host/etc/cni/net.d"
+    }
+```
+
+### Client / Shim configuration
+
+The multus shim configuration is encoded in JSON, and essentially is just a
+regular CNI configuration, usually available in `/etc/cni/net.d/00-multus.conf`.
+
+It allows the following keys:
+
+- `"cniVersion"`: the CNI version for the Multus CNI plugin.
+- `"logFile"`:  the path to where the daemon logs will be persisted.
+- `"logLevel"`: the logging level for the multus daemon logs.
+- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
+  as well. By default, it is disabled.
+
+#### Chroot configuration
+
+In thick plugin case, delegate CNI plugin is executed by multus-daemon from Pod, hence if the delegate CNI requires resources in container host, for example unix socket or even file, then CNI plugin is failed to execute because multus-daemon runs in Pod. Multus-daemon supports "chrootDir" option which executes delegate CNI under chroot (to container host).
+
+This configuration is enabled in deployments/multus-daemonset-thick.yml as default.

e2e/README.md

@@ -1,12 +1,20 @@
 ## Multus e2e test with kind
 
-### How to test e2e
+### Prerequisite
 
+To run the e2e test, you need the following components:
+
+- curl
+- j2cli
+- docker
+
+### How to test e2e
 
 ```
 $ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 $ cd multus-cni/e2e
 $ ./get_tools.sh
+$ ./generate_yamls.sh
 $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```

e2e/generate_yamls.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ ! -d yamls ]; then
+	mkdir yamls
+fi
+
+# specify CNI version (default: 0.4.0)
+export CNI_VERSION=${CNI_VERSION:-0.4.0}
+
+templates_dir="$(dirname $(readlink -f $0))/templates"
+
+# generate yaml files based on templates/*.j2 to yamls directory
+for i in `ls templates/`; do
+	echo $i
+	j2 -e CNI_VERSION ${templates_dir}/$i -o yamls/${i%.j2}
+done
+unset CNI_VERSION

e2e/multus-daemonset.yml

@@ -1,296 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              properties:
-                config:
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: localhost:5000/multus:e2e
-        imagePullPolicy: Always
-        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
-        args:
-        - "-multus-conf-file=auto"
-        - "-cni-version=0.3.1"
-        - "-cni-config-dir=/host/etc/cni/net.d"
-        - "-multus-autoconfig-dir=/host/etc/cni/net.d"
-        - "-multus-log-to-stderr=true"
-        - "-multus-log-level=debug"
-        - "-multus-log-file=/tmp/multus.log"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      initContainers:
-      - name: install-multus-binary
-        image: localhost:5000/multus:e2e
-        command:
-          - "cp"
-          - "/usr/src/multus-cni/bin/multus"
-          - "/host/opt/cni/bin/multus"
-        resources:
-          requests:
-            cpu: "10m"
-            memory: "15Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-          - name: cnibin
-            mountPath: /host/opt/cni/bin
-            mountPropagation: Bidirectional
-      - name: generate-kubeconfig
-        image: localhost:5000/multus:e2e
-        command:
-          - "/usr/src/multus-cni/bin/generate-kubeconfig"
-        args:
-          - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
-          - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
-        resources:
-          requests:
-            cpu: "10m"
-            memory: "15Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-          - name: cni
-            mountPath: /host/etc/cni/net.d
-            mountPropagation: Bidirectional
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # ppc64le support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

e2e/setup_cluster.sh

@@ -8,9 +8,9 @@ export PATH=${PATH}:./bin
 OCI_BIN="${OCI_BIN:-docker}"
 
 # define the deployment spec to use when deploying multus.
-# Acceptable values are `legacy-multus-daemonset.yml`. `multus-daemonset.yml`.
-# Defaults to `multus-daemonset.yml`.
-MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset.yml}"
+# Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
+# Defaults to `multus-daemonset-thick.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
 
 kind_network='kind'
 reg_name='kind-registry'
@@ -78,9 +78,9 @@ kind export kubeconfig
 sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
 sleep 1
 kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f "$MULTUS_MANIFEST"
+kubectl create -f yamls/$MULTUS_MANIFEST
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
-kubectl create -f cni-install.yml
+kubectl create -f yamls/cni-install.yml
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/templates/default-route1.yml.j2

@@ -5,7 +5,7 @@ metadata:
   name: default-route-config
 spec: 
   config: '{
-            "cniVersion": "0.3.1",
+            "cniVersion": "{{ CNI_VERSION }}",
             "plugins": [
                 {
                     "type": "macvlan",

e2e/templates/multus-daemonset-thick.yml.j2

@@ -0,0 +1,198 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "debug",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/host/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus",
+        "cniVersion": "{{ CNI_VERSION }}",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "forceCNIVersion": true,
+        "multusAutoconfigDir": "/host/etc/cni/net.d"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        imagePullPolicy: Always
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: host-run
+          mountPath: /host/run
+        - name: host-var-lib-cni-multus
+          mountPath: /var/lib/cni/multus
+        - name: host-run-netns
+          mountPath: /run/netns
+          mountPropagation: HostToContainer
+        - name: multus-daemon-config
+          mountPath: /etc/cni/net.d/multus.d
+          readOnly: true
+      initContainers:
+      - name: install-multus-shim
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus-shim"
+          - "/host/opt/cni/bin/multus-shim"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-daemon-config
+          configMap:
+            name: multus-daemon-config
+            items:
+            - key: daemon-config.json
+              path: daemon-config.json
+        - name: host-run
+          hostPath:
+            path: /run
+        - name: host-var-lib-cni-multus
+          hostPath:
+            path: /var/lib/cni/multus
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

e2e/templates/multus-daemonset.yml.j2

@@ -145,10 +145,11 @@ spec:
       containers:
       - name: kube-multus
         image: localhost:5000/multus:e2e
-        command: ["/entrypoint.sh"]
+        command: ["/thin_entrypoint"]
         args:
         - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
+        - "--force-cni-version=true"
+        - "--cni-version={{ CNI_VERSION }}"
         resources:
           requests:
             cpu: "100m"
@@ -168,10 +169,10 @@ spec:
       initContainers:
       - name: install-multus-binary
         image: localhost:5000/multus:e2e
-        command:
-          - "cp"
-          - "/usr/src/multus-cni/bin/multus"
-          - "/host/opt/cni/bin/multus"
+        command: ["/install_multus"]
+        args:
+          - "--type"
+          - "thin"
         resources:
           requests:
             cpu: "10m"
@@ -195,70 +196,3 @@ spec:
             items:
             - key: cni-conf.json
               path: 70-multus.conf
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # ppc64le support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--multus-conf-file=auto"
-        - "--cni-version=0.3.1"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

e2e/templates/simple-macvlan1.yml.j2

@@ -5,7 +5,7 @@ metadata:
   name: macvlan1-config
 spec: 
   config: '{
-            "cniVersion": "0.3.1",
+            "cniVersion": "{{ CNI_VERSION }}",
             "plugins": [
                 {
                     "type": "macvlan",

e2e/test-default-route1.sh

@@ -3,11 +3,14 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f default-route1.yml
+kubectl create -f yamls/default-route1.yml
 kubectl wait --for=condition=ready -l app=default-route1 --timeout=300s pod
 
 echo "check default-route-worker1 interface: net1"
 kubectl exec default-route-worker1 -- ip a show dev net1
+if [ $? -ne 0 ];then
+       exit 1
+fi
 
 echo "check default-route-worker1 interface address: net1"
 ipaddr=$(kubectl exec default-route-worker1 -- ip -j a show  | jq -r \
@@ -25,6 +28,9 @@ fi
 
 echo "check default-route-worker2 interface: net1"
 kubectl exec default-route-worker2 -- ip a show dev net1
+if [ $? -ne 0 ];then
+       exit 1
+fi
 
 echo "check default-route-worker2 interface address: net1"
 ipaddr=$(kubectl exec default-route-worker2 -- ip -j a show  | jq -r \
@@ -41,4 +47,4 @@ if [ $ipaddr != "10.244.1.1" ]; then
 fi
 
 echo "cleanup resources"
-kubectl delete -f default-route1.yml
+kubectl delete -f yamls/default-route1.yml

e2e/test-simple-macvlan1.sh

@@ -3,11 +3,17 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f simple-macvlan1.yml
+kubectl create -f yamls/simple-macvlan1.yml
 kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
 
 echo "check macvlan1-worker1 interface: net1"
-kubectl exec macvlan1-worker1 -- ip a show dev net1
+net=$(kubectl exec macvlan1-worker1 -- ip a show dev net1)
+if [ $? -eq 0 ];then
+	echo "macvlan1-worker1 pod has net1 card"
+else
+	echo "macvlan1-worker1 pod has no net1 card"
+	exit 1
+fi
 
 echo "check macvlan1-worker1 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
@@ -17,7 +23,13 @@ if [ $ipaddr != "10.1.1.11" ]; then
 fi
 
 echo "check macvlan1-worker2 interface: net1"
-kubectl exec macvlan1-worker2 -- ip a show dev net1
+net2=$(kubectl exec macvlan1-worker2 -- ip a show dev net1)
+if [ $? -eq 0  ];then
+	echo "macvlan1-worker2 pod has net1 card"
+else
+	echo "macvlan1-worker2 pod has no net1 card"
+	exit 1
+fi
 
 echo "check macvlan1-worker2 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
@@ -27,4 +39,4 @@ if [ $ipaddr != "10.1.1.12" ]; then
 fi
 
 echo "cleanup resources"
-kubectl delete -f simple-macvlan1.yml
+kubectl delete -f yamls/simple-macvlan1.yml

e2e/test-simple-pod.sh

@@ -3,8 +3,8 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f simple-pod.yml
+kubectl create -f yamls/simple-pod.yml
 kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
 
 echo "cleanup resources"
-kubectl delete -f simple-pod.yml
+kubectl delete -f yamls/simple-pod.yml

go.mod

@@ -1,53 +1,96 @@
-module gopkg.in/k8snetworkplumbingwg/multus-cni.v3
+module gopkg.in/k8snetworkplumbingwg/multus-cni.v4
 
-go 1.16
+go 1.18
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/containernetworking/cni v0.8.1
-	github.com/containernetworking/plugins v0.9.1
-	github.com/fsnotify/fsnotify v1.4.9
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe
-	github.com/onsi/ginkgo v1.12.1
-	github.com/onsi/gomega v1.10.3
-	github.com/pkg/errors v0.9.1
-	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
-	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
-	google.golang.org/grpc v1.27.1
+	github.com/containernetworking/cni v1.1.2
+	github.com/containernetworking/plugins v1.1.0
+	github.com/fsnotify/fsnotify v1.6.0
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
+	github.com/onsi/ginkgo/v2 v2.5.1
+	github.com/onsi/gomega v1.24.0
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
+	golang.org/x/net v0.7.0
+	golang.org/x/sys v0.5.0
+	google.golang.org/grpc v1.40.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	k8s.io/api v0.20.10
-	k8s.io/apimachinery v0.20.10
-	k8s.io/client-go v0.20.10
+	k8s.io/api v0.22.8
+	k8s.io/apimachinery v0.22.8
+	k8s.io/client-go v0.22.8
 	k8s.io/klog v1.0.0
-	k8s.io/kubelet v0.0.0
-	k8s.io/kubernetes v1.20.10
+	k8s.io/klog/v2 v2.60.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 // indirect
+	k8s.io/kubelet v0.22.8
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+require github.com/prometheus/client_golang v1.12.2
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 )
 
 replace (
 	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
-	k8s.io/api => k8s.io/api v0.20.10
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.10
-	k8s.io/apimachinery => k8s.io/apimachinery v0.20.10
-	k8s.io/apiserver => k8s.io/apiserver v0.20.10
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.10
-	k8s.io/client-go => k8s.io/client-go v0.20.10
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.10
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.10
-	k8s.io/code-generator => k8s.io/code-generator v0.20.10
-	k8s.io/component-base => k8s.io/component-base v0.20.10
-	k8s.io/component-helpers => k8s.io/component-helpers v0.20.10
-	k8s.io/controller-manager => k8s.io/controller-manager v0.20.10
-	k8s.io/cri-api => k8s.io/cri-api v0.20.10
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.10
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.10
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.10
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.10
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.10
-	k8s.io/kubectl => k8s.io/kubectl v0.20.10
-	k8s.io/kubelet => k8s.io/kubelet v0.20.10
-	k8s.io/kubernetes => k8s.io/kubernetes v1.20.10
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.10
-	k8s.io/metrics => k8s.io/metrics v0.20.10
-	k8s.io/mount-utils => k8s.io/mount-utils v0.20.10
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.10
+	k8s.io/api => k8s.io/api v0.22.8
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.8
+	k8s.io/apimachinery => k8s.io/apimachinery v0.22.8
+	k8s.io/apiserver => k8s.io/apiserver v0.22.8
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.22.8
+	k8s.io/client-go => k8s.io/client-go v0.22.8
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.22.8
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.8
+	k8s.io/code-generator => k8s.io/code-generator v0.22.8
+	k8s.io/component-base => k8s.io/component-base v0.22.8
+	k8s.io/component-helpers => k8s.io/component-helpers v0.22.8
+	k8s.io/controller-manager => k8s.io/controller-manager v0.22.8
+	k8s.io/cri-api => k8s.io/cri-api v0.22.8
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.22.8
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.22.8
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.22.8
+	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.22.8
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.22.8
+	k8s.io/kubectl => k8s.io/kubectl v0.22.8
+	k8s.io/kubelet => k8s.io/kubelet v0.22.8
+	k8s.io/kubernetes => k8s.io/kubernetes v1.22.8
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.22.8
+	k8s.io/metrics => k8s.io/metrics v0.22.8
+	k8s.io/mount-utils => k8s.io/mount-utils v0.22.8
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.8
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.22.8
 )

hack/build-go.sh

@@ -7,54 +7,72 @@ if [ ! -d ${DEST_DIR} ]; then
 	mkdir ${DEST_DIR}
 fi
 
-# Add version/commit/date into binary
-# In case of TravisCI, need to check error code of 'git describe'.
-if [ -z "$VERSION" ]; then
+# Specify correspondingGOARCH from TARGETPLATFORM
+if [ "$TARGETPLATFORM" = "linux/amd64" ]; then
+	export GOARCH=amd64
+elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then
+	export GOARCH=arm64
+elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then
+	export GOARCH=arm
+elif [ "$TARGETPLATFORM" = "linux/ppc64le" ]; then
+	export GOARCH=ppc64le
+elif [ "$TARGETPLATFORM" = "linux/s390x" ]; then
+	export GOARCH=s390x
+fi
+
+# version information
+hasGit=true
+git version > /dev/null 2>&1 || hasGit=false
+GIT_SHA=""
+GIT_TREE_STATE=""
+GIT_TAG=""
+GIT_TAG_LAST=""
+RELEASE_STATUS=""
+if $hasGit; then
 	set +e
-	git describe --tags --abbrev=0 > /dev/null 2>&1
-	if [ "$?" != "0" ]; then
-		VERSION="master"
-	else
-		VERSION=$(git describe --tags --abbrev=0)
-	fi
+	GIT_SHA=$(git rev-parse --short HEAD)
+	# Tree state is "dirty" if there are uncommitted changes, untracked files are ignored
+	GIT_TREE_STATE=$(test -n "`git status --porcelain --untracked-files=no`" && echo "dirty" || echo "clean")
+	# Empty string if we are not building a tag
+	GIT_TAG=$(git describe --tags --abbrev=0 --exact-match 2>/dev/null || true)
+	# Find most recent tag
+	GIT_TAG_LAST=$(git describe --tags --abbrev=0 2>/dev/null || true)
 	set -e
 fi
+
+# VERSION override mechanism if needed
+VERSION=${VERSION:-}
+if [[ -n "${VERSION}" || -n "${GIT_TAG}" ]]; then
+	RELEASE_STATUS=",released"
+fi
+
+if [ -z "$VERSION" ]; then
+	VERSION=$GIT_TAG_LAST
+fi
+# Add version/commit/date into binary
 DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
 COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
-LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version=${VERSION:-master} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit=${COMMIT} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date=${DATE}"
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version=${VERSION} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit=${COMMIT} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.gitTreeState=${GIT_TREE_STATE} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.releaseStatus=${RELEASE_STATUS} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date=${DATE}"
 export CGO_ENABLED=0
 
-# this if... will be removed when gomodules goes default
-if [ "$GO111MODULE" == "off" ]; then
-	echo "Building plugin without go module"
-	echo "Warning: this will be deprecated in near future so please use go modules!"
-
-	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
-	REPO_PATH="${ORG_PATH}/multus-cni.v3"
-
-	if [ ! -h gopath/src/${REPO_PATH} ]; then
-		mkdir -p gopath/src/${ORG_PATH}
-		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
-	fi
-
-	export GO15VENDOREXPERIMENT=1
-	export GOBIN=${PWD}/bin
-	export GOPATH=${PWD}/gopath
-	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
-	go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/config-generation
-	go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/controller/
-else
-	# build with go modules
-	export GO111MODULE=on
-	BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
-	if [ -n "$MODMODE" ]; then
-		BUILD_ARGS+=(-mod "$MODMODE")
-	fi
-
-	echo "Building plugins"
-	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
-	echo "Building spec generators"
-	go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/config-generation
-	echo "Building multus controller"
-	go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/controller/
+# build with go modules
+export GO111MODULE=on
+BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+if [ -n "$MODMODE" ]; then
+	BUILD_ARGS+=(-mod "$MODMODE")
 fi
+
+echo "Building multus"
+go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
+echo "Building multus-daemon"
+go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/multus-daemon
+echo "Building multus-shim"
+go build -o "${DEST_DIR}"/multus-shim -ldflags "${LDFLAGS}" ./cmd/multus-shim
+echo "Building install_multus"
+go build -o "${DEST_DIR}"/install_multus -ldflags "${LDFLAGS}" ./cmd/install_multus
+echo "Building thin_entrypoint"
+go build -o "${DEST_DIR}"/thin_entrypoint -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint

images/Dockerfile

@@ -1,17 +1,19 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
+FROM --platform=$BUILDPLATFORM golang:1.19 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
+ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM centos:centos7
+FROM gcr.io/distroless/base-debian11:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 WORKDIR /
 
-ADD ./images/entrypoint.sh /
-ENTRYPOINT ["/entrypoint.sh"]
+COPY --from=build /usr/src/multus-cni/bin/install_multus /
+COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.arm64

@@ -1,22 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV GOARCH "arm64"
-ENV GOOS "linux"
-
-RUN  cd /usr/src/multus-cni && \
-     ./hack/build-go.sh
-
-# build arm64 container
-FROM arm64v8/centos:7
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
-COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-
-WORKDIR /
-ADD ./images/entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.debug

@@ -1,22 +1,19 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
+FROM --platform=$BUILDPLATFORM golang:1.19 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
-ENV GOARCH "arm"
-ENV GOOS "linux"
-
+ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-# build arm container
-FROM arm32v7/centos:7
+FROM gcr.io/distroless/base-debian11:debug
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-
 WORKDIR /
-ADD ./images/entrypoint.sh /
 
-ENTRYPOINT ["/entrypoint.sh"]
+COPY --from=build /usr/src/multus-cni/bin/install_multus /
+COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.openshift

@@ -1,5 +1,6 @@
 # This dockerfile is specific to building Multus for OpenShift
-FROM openshift/origin-release:golang-1.16 as builder
+# The okd-builder image is locally built from https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile
+FROM local/okdbuilder:latest as builder
 
 ADD . /usr/src/multus-cni
 
@@ -7,15 +8,16 @@ WORKDIR /usr/src/multus-cni
 ENV GO111MODULE=off
 RUN ./hack/build-go.sh
 
-FROM openshift/origin-base
+FROM quay.io/openshift/origin-base:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
 COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
-ADD ./images/entrypoint.sh /
+COPY --from=builder /usr/src/multus-cni/bin/install_multus /
+COPY --from=builder /usr/src/multus-cni/bin/thin_entrypoint /
 
 LABEL io.k8s.display-name="Multus CNI" \
       io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
       io.openshift.tags="openshift" \
       maintainer="Doug Smith <dosmith@redhat.com>"
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.ppc64le

@@ -1,22 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV GOARCH "ppc64le"
-ENV GOOS "linux"
-
-RUN  cd /usr/src/multus-cni && \
-     ./hack/build-go.sh
-
-# build ppc container
-FROM ppc64le/centos:latest
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
-COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-
-WORKDIR /
-ADD ./images/entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.s390x

@@ -1,21 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV GOARCH "s390x"
-ENV GOOS "linux"
-
-RUN  cd /usr/src/multus-cni && \
-     ./hack/build-go.sh
-
-# build s390x container
-FROM s390x/python:3-slim
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
-COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-WORKDIR /
-ADD ./images/entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.thick

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.17.9 as build
+FROM golang:1.19 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -7,7 +7,7 @@ ADD . /usr/src/multus-cni
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal
+FROM debian:stable-slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE

images/entrypoint.sh

@@ -1,479 +0,0 @@
-#!/bin/bash
-
-# Always exit on errors.
-set -e
-
-# Trap sigterm
-function exitonsigterm() {
-  echo "Trapped sigterm, exiting."
-  exit 0
-}
-trap exitonsigterm SIGTERM
-
-# Set our known directories.
-CNI_CONF_DIR="/host/etc/cni/net.d"
-CNI_BIN_DIR="/host/opt/cni/bin"
-ADDITIONAL_BIN_DIR=""
-MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
-MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
-MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
-MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
-MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
-MULTUS_MASTER_CNI_FILE_NAME=""
-MULTUS_NAMESPACE_ISOLATION=false
-MULTUS_GLOBAL_NAMESPACES=""
-MULTUS_LOG_TO_STDERR=true
-MULTUS_LOG_LEVEL=""
-MULTUS_LOG_FILE=""
-MULTUS_READINESS_INDICATOR_FILE=""
-OVERRIDE_NETWORK_NAME=false
-MULTUS_CLEANUP_CONFIG_ON_EXIT=false
-RESTART_CRIO=false
-CRIO_RESTARTED_ONCE=false
-RENAME_SOURCE_CONFIG_FILE=false
-SKIP_BINARY_COPY=false
-
-# Give help text for parameters.
-function usage()
-{
-    echo -e "This is an entrypoint script for Multus CNI to overlay its configuration into"
-    echo -e "locations in a filesystem. The configuration file will be copied to the"
-    echo -e "corresponding configuration directory. When '--multus-conf-file=auto' is used,"
-    echo -e "00-multus.conf will be automatically generated from the CNI configuration file"
-    echo -e "of the master plugin (the first file in lexicographical order in cni-conf-dir)."
-    echo -e "When '--multus-master-cni-file-name' is used, 00-multus.conf will be"
-    echo -e "automatically generated from the specific file rather than the first file."
-    echo -e ""
-    echo -e "./entrypoint.sh"
-    echo -e "\t-h --help"
-    echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
-    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
-    echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
-    echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
-    echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
-    echo -e "\t--skip-multus-binary-copy=$SKIP_BINARY_COPY"
-    echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
-    echo -e "\t--multus-master-cni-file-name=$MULTUS_MASTER_CNI_FILE_NAME (empty by default, example: 10-calico.conflist)"
-    echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
-    echo -e "\t--global-namespaces=$MULTUS_GLOBAL_NAMESPACES (used only with --namespace-isolation=true)"
-    echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
-    echo -e "\t--multus-log-to-stderr=$MULTUS_LOG_TO_STDERR (empty by default, used only with --multus-conf-file=auto)"
-    echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
-    echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
-    echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--readiness-indicator-file=$MULTUS_READINESS_INDICATOR_FILE (used only with --multus-conf-file=auto)"
-    echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)"
-    echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)"
-}
-
-function log()
-{
-    echo "$(date --iso-8601=seconds) ${1}"
-}
-
-function error()
-{
-    log "ERR:  {$1}"
-}
-
-function warn()
-{
-    log "WARN: {$1}"
-}
-
-if type python3 &> /dev/null; then
-	alias python=python3
-fi
-
-function checkCniVersion {
-      cniversion_python_tmpfile=$(mktemp)
-      cat << EOF > $cniversion_python_tmpfile
-import json, sys
-
-def version(v):
-    return [int(x) for x in v.split(".")]
-
-v_040 = version("0.4.0")
-v_top_level = sys.argv[2]
-with open(sys.argv[1], "r") as f:
-    v_nested = json.load(f)["cniVersion"]
-if version(v_top_level) >= v_040 and version(v_nested) < v_040:
-    msg = "Multus cni version is %s while master plugin cni version is %s"
-    print(msg % (v_top_level, v_nested))
-EOF
-      python $cniversion_python_tmpfile $1 $2
-}
-
-# Parse parameters given as arguments to this script.
-while [ "$1" != "" ]; do
-    PARAM=`echo $1 | awk -F= '{print $1}'`
-    VALUE=`echo $1 | awk -F= '{print $2}'`
-    case $PARAM in
-        -h | --help)
-            usage
-            exit
-            ;;
-        --cni-version)
-            CNI_VERSION=$VALUE
-            ;;
-        --cni-bin-dir)
-            CNI_BIN_DIR=$VALUE
-            ;;
-        --cni-conf-dir)
-            CNI_CONF_DIR=$VALUE
-            ;;
-        --cni-bin-dir)
-            CNI_BIN_DIR=$VALUE
-            ;;
-        --multus-conf-file)
-            MULTUS_CONF_FILE=$VALUE
-            ;;
-        --multus-kubeconfig-file-host)
-            MULTUS_KUBECONFIG_FILE_HOST=$VALUE
-            ;;
-        --multus-master-cni-file-name)
-            MULTUS_MASTER_CNI_FILE_NAME=$VALUE
-            ;;
-        --namespace-isolation)
-            MULTUS_NAMESPACE_ISOLATION=$VALUE
-            ;;
-        --global-namespaces)
-            MULTUS_GLOBAL_NAMESPACES=$VALUE
-            ;;
-        --multus-log-to-stderr)
-            MULTUS_LOG_TO_STDERR=$VALUE
-            ;;
-        --multus-log-level)
-            MULTUS_LOG_LEVEL=$VALUE
-            ;;
-        --multus-log-file)
-            MULTUS_LOG_FILE=$VALUE
-            ;;
-        --multus-autoconfig-dir)
-            MULTUS_AUTOCONF_DIR=$VALUE
-            ;;
-        --override-network-name)
-            OVERRIDE_NETWORK_NAME=$VALUE
-            ;;
-        --cleanup-config-on-exit)
-            MULTUS_CLEANUP_CONFIG_ON_EXIT=$VALUE
-            ;;
-        --restart-crio)
-            RESTART_CRIO=$VALUE
-            ;;
-        --rename-conf-file)
-            RENAME_SOURCE_CONFIG_FILE=$VALUE
-            ;;
-        --additional-bin-dir)
-            ADDITIONAL_BIN_DIR=$VALUE
-            ;;
-        --skip-multus-binary-copy)
-            SKIP_BINARY_COPY=$VALUE
-            ;;
-        --readiness-indicator-file)
-            MULTUS_READINESS_INDICATOR_FILE=$VALUE
-            ;;
-        *)
-            warn "unknown parameter \"$PARAM\""
-            ;;
-    esac
-    shift
-done
-
-
-# Create array of known locations
-declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE)
-if [ "$MULTUS_CONF_FILE" != "auto" ]; then
-  arr+=($MULTUS_CONF_FILE)
-fi
-
-
-# Loop through and verify each location each.
-for i in "${arr[@]}"
-do
-  if [ ! -e "$i" ]; then
-    warn "Location $i does not exist"
-    exit 1;
-  fi
-done
-
-# Copy files into place and atomically move into final binary name
-if [ "$SKIP_BINARY_COPY" = false ]; then
-  cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR/_multus
-  mv -f $CNI_BIN_DIR/_multus $CNI_BIN_DIR/multus
-else
-  log "Entrypoint skipped copying Multus binary."
-fi
-
-if [ "$MULTUS_CONF_FILE" != "auto" ]; then
-  cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
-fi
-
-# Make a multus.d directory (for our kubeconfig)
-
-mkdir -p $CNI_CONF_DIR/multus.d
-MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
-
-# ------------------------------- Generate a "kube-config"
-# Inspired by: https://tinyurl.com/y7r2knme
-SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
-KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
-SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
-SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
-
-
-# Check if we're running as a k8s pod.
-if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
-  # We're running as a k8d pod - expect some variables.
-  if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
-    error "KUBERNETES_SERVICE_HOST not set"; exit 1;
-  fi
-  if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
-    error "KUBERNETES_SERVICE_PORT not set"; exit 1;
-  fi
-
-  if [ "$SKIP_TLS_VERIFY" == "true" ]; then
-    TLS_CFG="insecure-skip-tls-verify: true"
-  elif [ -f "$KUBE_CA_FILE" ]; then
-    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
-  fi
-
-  # Write a kubeconfig file for the CNI plugin.  Do this
-  # to skip TLS verification for now.  We should eventually support
-  # writing more complete kubeconfig files. This is only used
-  # if the provided CNI network config references it.
-  touch $MULTUS_TEMP_KUBECONFIG
-  chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
-  # Write the kubeconfig to a temp file first.
-  cat > $MULTUS_TEMP_KUBECONFIG <<EOF
-# Kubeconfig file for Multus CNI plugin.
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT}
-    $TLS_CFG
-users:
-- name: multus
-  user:
-    token: "${SERVICEACCOUNT_TOKEN}"
-contexts:
-- name: multus-context
-  context:
-    cluster: local
-    user: multus
-current-context: multus-context
-EOF
-
-  # Atomically move the temp kubeconfig to its permanent home.
-  mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG
-
-else
-  warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
-fi
-
-# ---------------------- end Generate a "kube-config".
-
-# ------------------------------- Generate "00-multus.conf"
-
-function generateMultusConf {
-if [ "$MULTUS_CONF_FILE" == "auto" ]; then
-  log "Generating Multus configuration file using files in $MULTUS_AUTOCONF_DIR..."
-  found_master=false
-  tries=0
-  while [ $found_master == false ]; do
-    if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then
-        MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME"
-    else
-        MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
-    fi
-    if [ "$MASTER_PLUGIN" == "" ]; then
-      if [ $tries -lt 600 ]; then
-        if ! (($tries % 5)); then
-          log "Attempting to find master plugin configuration, attempt $tries"
-        fi
-        let "tries+=1"
-        sleep 1;
-      else
-        error "Multus could not be configured: no master plugin was found."
-        exit 1;
-      fi
-    else
-      log "Using MASTER_PLUGIN: $MASTER_PLUGIN"
-
-      found_master=true
-
-      ISOLATION_STRING=""
-      if [ "$MULTUS_NAMESPACE_ISOLATION" == true ]; then
-        ISOLATION_STRING="\"namespaceIsolation\": true,"
-      fi
-
-      GLOBAL_NAMESPACES_STRING=""
-      if [ ! -z "${MULTUS_GLOBAL_NAMESPACES// }" ]; then
-        GLOBAL_NAMESPACES_STRING="\"globalNamespaces\": \"$MULTUS_GLOBAL_NAMESPACES\","
-      fi
-
-      LOG_TO_STDERR_STRING=""
-      if [ "$MULTUS_LOG_TO_STDERR" == false ]; then
-        LOG_TO_STDERR_STRING="\"logToStderr\": false,"
-      fi
-
-
-      LOG_LEVEL_STRING=""
-      if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
-        case "$MULTUS_LOG_LEVEL" in
-          debug)
-              ;;
-          error)
-              ;;
-          panic)
-              ;;
-          verbose)
-              ;;
-          *)
-              error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
-              usage
-              exit 1
-        esac
-        LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
-      fi
-
-      LOG_FILE_STRING=""
-      if [ ! -z "${MULTUS_LOG_FILE// }" ]; then
-        LOG_FILE_STRING="\"logFile\": \"$MULTUS_LOG_FILE\","
-      fi
-
-      CNI_VERSION_STRING=""
-      if [ ! -z "${CNI_VERSION// }" ]; then
-        CNI_VERSION_STRING="\"cniVersion\": \"$CNI_VERSION\","
-      fi
-
-      ADDITIONAL_BIN_DIR_STRING=""
-      if [ ! -z "${ADDITIONAL_BIN_DIR// }" ]; then
-        ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\","
-      fi
-
-
-      READINESS_INDICATOR_FILE_STRING=""
-      if [ ! -z "${MULTUS_READINESS_INDICATOR_FILE// }" ]; then
-        READINESS_INDICATOR_FILE_STRING="\"readinessindicatorfile\": \"$MULTUS_READINESS_INDICATOR_FILE\","
-      fi
-
-      if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
-        MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
-      else
-        MASTER_PLUGIN_NET_NAME="multus-cni-network"
-      fi
-
-      capabilities_python_filter_tmpfile=$(mktemp)
-      cat << EOF > $capabilities_python_filter_tmpfile
-import json,sys
-conf = json.load(sys.stdin)
-capabilities = {}
-if 'plugins' in conf:
-    for capa in [p['capabilities'] for p in conf['plugins'] if 'capabilities' in p]:
-        capabilities.update({capability:enabled for (capability,enabled) in capa.items() if enabled})
-elif 'capabilities' in conf:
-    capabilities.update({capability:enabled for (capability,enabled) in conf['capabilities'] if enabled})
-if len(capabilities) > 0:
-    print("""\"capabilities\": """ + json.dumps(capabilities) + ",")
-else:
-    print("")
-EOF
-
-      NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python $capabilities_python_filter_tmpfile)"
-      rm $capabilities_python_filter_tmpfile
-      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING"
-
-      MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
-      MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
-      log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
-      CHECK_CNI_VERSION=$(checkCniVersion $MASTER_PLUGIN_LOCATION $CNI_VERSION)
-      if [ "$CHECK_CNI_VERSION" != "" ] ; then
-        error "$CHECK_CNI_VERSION"
-        exit 1
-      fi
-
-      CONF=$(cat <<-EOF
-        {
-          $CNI_VERSION_STRING
-          "name": "$MASTER_PLUGIN_NET_NAME",
-          "type": "multus",
-          $NESTED_CAPABILITIES_STRING
-          $ISOLATION_STRING
-          $GLOBAL_NAMESPACES_STRING
-          $LOG_TO_STDERR_STRING
-          $LOG_LEVEL_STRING
-          $LOG_FILE_STRING
-          $ADDITIONAL_BIN_DIR_STRING
-          $READINESS_INDICATOR_FILE_STRING
-          "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
-          "delegates": [
-            $MASTER_PLUGIN_JSON
-          ]
-        }
-EOF
-      )
-      tmpfile=$(mktemp)
-      echo $CONF > $tmpfile
-      mv $tmpfile $CNI_CONF_DIR/00-multus.conf
-      log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
-      echo $CONF
-
-      # If we're not performing the cleanup on exit, we can safely rename the config file.
-      if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
-        mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old
-        log "Original master file moved to ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old"
-      fi
-
-      if [ "$RESTART_CRIO" == true ]; then
-        # Restart CRIO only once.
-        if [ "$CRIO_RESTARTED_ONCE" == false ]; then
-          log "Restarting crio"
-          systemctl restart crio
-          CRIO_RESTARTED_ONCE=true
-        fi
-      fi
-    fi
-  done
-fi
-}
-generateMultusConf
-
-# ---------------------- end Generate "00-multus.conf".
-
-# Enter either sleep loop, or watch loop...
-if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
-  log "Entering watch loop..."
-  while true; do
-    # Check and see if the original master plugin configuration exists...
-    if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then
-      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Allowing 45 seconds for its restoration..."
-      sleep 10
-      for i in {1..35}
-      do
-        if [ -f "$MASTER_PLUGIN_LOCATION" ]; then
-          log "Master plugin @ $MASTER_PLUGIN_LOCATION was restored. Regenerating given configuration."
-          break
-        fi
-        sleep 1
-      done
-
-      generateMultusConf
-      log "Continuing watch loop after configuration regeneration..."
-    fi
-    sleep 1
-  done
-else
-  log "Entering sleep (success)..."
-  if tty -s; then
-	  read
-  else
-	  sleep infinity
-  fi
-fi

pkg/checkpoint/checkpoint.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,16 +12,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 package checkpoint
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"os"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -33,7 +33,7 @@ type PodDevicesEntry struct {
 	PodUID        string
 	ContainerName string
 	ResourceName  string
-	DeviceIDs     []string
+	DeviceIDs     map[int64][]string
 	AllocResp     []byte
 }
 
@@ -72,7 +72,7 @@ func getCheckpoint(filePath string) (types.ResourceClient, error) {
 func (cp *checkpoint) getPodEntries() error {
 
 	cpd := &checkpointFileData{}
-	rawBytes, err := ioutil.ReadFile(cp.fileName)
+	rawBytes, err := os.ReadFile(cp.fileName)
 	if err != nil {
 		return logging.Errorf("getPodEntries: error reading file %s\n%v\n", checkPointfile, err)
 	}
@@ -86,7 +86,7 @@ func (cp *checkpoint) getPodEntries() error {
 	return nil
 }
 
-// GetComputeDeviceMap returns an instance of a map of ResourceInfo
+// GetPodResourceMap returns an instance of a map of ResourceInfo
 func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
 	podID := string(pod.UID)
 	resourceMap := make(map[string]*types.ResourceInfo)
@@ -97,12 +97,14 @@ func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resource
 	for _, pod := range cp.podEntires {
 		if pod.PodUID == podID {
 			entry, ok := resourceMap[pod.ResourceName]
-			if ok {
-				// already exists; append to it
-				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
-			} else {
+			if !ok {
 				// new entry
-				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
+				entry = &types.ResourceInfo{}
+				resourceMap[pod.ResourceName] = entry
+			}
+			for _, v := range pod.DeviceIDs {
+				// already exists; append to it
+				entry.DeviceIDs = append(entry.DeviceIDs, v...)
 			}
 		}
 	}

pkg/checkpoint/checkpoint_test.go

@@ -1,16 +1,30 @@
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package checkpoint
 
 import (
 	"fmt"
 	"os"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"io/ioutil"
 	"testing"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
@@ -25,7 +39,7 @@ type fakeCheckpoint struct {
 }
 
 func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error {
-	return ioutil.WriteFile(fc.fileName, inBytes, 0600)
+	return os.WriteFile(fc.fileName, inBytes, 0600)
 }
 
 func (fc *fakeCheckpoint) DeleteFile() error {
@@ -45,10 +59,11 @@ var _ = BeforeSuite(func() {
 				"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
 				"ContainerName": "appcntr1",
 				"ResourceName": "intel.com/sriov_net_A",
-				"DeviceIDs": [
-				"0000:03:02.3",
-				"0000:03:02.0"
-				],
+				"DeviceIDs": {"-1": [
+					"0000:03:02.3",
+					"0000:03:02.0"
+					]
+				},
 				"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
 			}
 			],
@@ -143,10 +158,10 @@ var _ = Describe("Kubelet checkpoint data read operations", func() {
 						"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
 						"ContainerName": "appcntr1",
 						"ResourceName": "intel.com/sriov_net_A",
-						"DeviceIDs": [
+						"DeviceIDs": { "-1": [
 						"0000:03:02.3",
 						"0000:03:02.0"
-						],
+						] },
 						"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
 					}
 					],

pkg/checkpoint/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/cmdutils/cmdutils_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cmdutils is the package that contains utilities for multus command
+package cmdutils
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestServer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "cmdutils")
+}

pkg/cmdutils/utils.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cmdutils is the package that contains utilities for multus command
+package cmdutils
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// CopyFileAtomic does file copy atomically
+func CopyFileAtomic(srcFilePath, destDir, tempFileName, destFileName string) error {
+	tempFilePath := filepath.Join(destDir, tempFileName)
+	// check temp filepath and remove old file if exists
+	if _, err := os.Stat(tempFilePath); err == nil {
+		err = os.Remove(tempFilePath)
+		if err != nil {
+			return fmt.Errorf("cannot remove old temp file %q: %v", tempFilePath, err)
+		}
+	}
+
+	// create temp file
+	f, err := os.CreateTemp(destDir, tempFileName)
+	defer f.Close()
+	if err != nil {
+		return fmt.Errorf("cannot create temp file %q in %q: %v", tempFileName, destDir, err)
+	}
+
+	srcFile, err := os.Open(srcFilePath)
+	if err != nil {
+		return fmt.Errorf("cannot open file %q: %v", srcFilePath, err)
+	}
+	defer srcFile.Close()
+
+	// Copy file to tempfile
+	_, err = io.Copy(f, srcFile)
+	if err != nil {
+		f.Close()
+		os.Remove(tempFilePath)
+		return fmt.Errorf("cannot write data to temp file %q: %v", tempFilePath, err)
+	}
+	if err := f.Sync(); err != nil {
+		return fmt.Errorf("cannot flush temp file %q: %v", tempFilePath, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("cannot close temp file %q: %v", tempFilePath, err)
+	}
+
+	// change file mode if different
+	destFilePath := filepath.Join(destDir, destFileName)
+	_, err = os.Stat(destFilePath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	srcFileStat, err := os.Stat(srcFilePath)
+	if err != nil {
+		return err
+	}
+
+	if err := os.Chmod(f.Name(), srcFileStat.Mode()); err != nil {
+		return fmt.Errorf("cannot set stat on temp file %q: %v", f.Name(), err)
+	}
+
+	// replace file with tempfile
+	if err := os.Rename(f.Name(), destFilePath); err != nil {
+		return fmt.Errorf("cannot replace %q with temp file %q: %v", destFilePath, tempFilePath, err)
+	}
+
+	return nil
+}

pkg/cmdutils/utils_test.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cmdutils is the package that contains utilities for multus command
+package cmdutils
+
+import (
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("thin entrypoint testing", func() {
+	It("Run CopyFileAtomic()", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		// create source directory
+		srcDir := fmt.Sprintf("%s/src", tmpDir)
+		err = os.Mkdir(srcDir, 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		// create destination directory
+		destDir := fmt.Sprintf("%s/dest", tmpDir)
+		err = os.Mkdir(destDir, 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		// sample source file
+		srcFilePath := fmt.Sprintf("%s/sampleInput", srcDir)
+		err = os.WriteFile(srcFilePath, []byte("sampleInputABC"), 0744)
+		Expect(err).NotTo(HaveOccurred())
+
+		// old files in dest
+		destFileName := "sampleInputDest"
+		destFilePath := fmt.Sprintf("%s/%s", destDir, destFileName)
+		err = os.WriteFile(destFilePath, []byte("inputOldXYZ"), 0611)
+		Expect(err).NotTo(HaveOccurred())
+
+		tempFileName := "temp_file"
+		err = CopyFileAtomic(srcFilePath, destDir, tempFileName, destFileName)
+		Expect(err).NotTo(HaveOccurred())
+
+		// check file mode
+		stat, err := os.Stat(destFilePath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(stat.Mode()).To(Equal(os.FileMode(0744)))
+
+		// check file contents
+		destFileByte, err := os.ReadFile(destFilePath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(destFileByte).To(Equal([]byte("sampleInputABC")))
+
+		err = os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+})

pkg/config/generator.go

@@ -1,333 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package config
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/blang/semver"
-)
-
-const (
-	configListCapabilityKey   = "plugins"
-	singleConfigCapabilityKey = "capabilities"
-)
-
-// LogOptionFunc mutates the `LoggingOptions` object
-type LogOptionFunc func(logOptions *LogOptions)
-
-// Option mutates the `conf` object
-type Option func(conf *MultusConf)
-
-// MultusConf holds the multus configuration, and persists it to disk
-type MultusConf struct {
-	BinDir                   string          `json:"binDir,omitempty"`
-	Capabilities             map[string]bool `json:"capabilities,omitempty"`
-	CNIVersion               string          `json:"cniVersion"`
-	Delegates                []interface{}   `json:"delegates"`
-	LogFile                  string          `json:"logFile,omitempty"`
-	LogLevel                 string          `json:"logLevel,omitempty"`
-	LogToStderr              bool            `json:"logToStderr,omitempty"`
-	LogOptions               *LogOptions     `json:"logOptions,omitempty"`
-	Kubeconfig               string          `json:"kubeconfig"`
-	Name                     string          `json:"name"`
-	NamespaceIsolation       bool            `json:"namespaceIsolation,omitempty"`
-	RawNonIsolatedNamespaces string          `json:"globalNamespaces,omitempty"`
-	ReadinessIndicatorFile   string          `json:"readinessindicatorfile,omitempty"`
-	Type                     string          `json:"type"`
-}
-
-// LogOptions specifies the configuration of the log
-type LogOptions struct {
-	MaxAge     *int  `json:"maxAge,omitempty"`
-	MaxSize    *int  `json:"maxSize,omitempty"`
-	MaxBackups *int  `json:"maxBackups,omitempty"`
-	Compress   *bool `json:"compress,omitempty"`
-}
-
-// NewMultusConfig creates a basic configuration generator. It can be mutated
-// via the `With...` methods.
-func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
-	multusConfig := &MultusConf{
-		Name:         MultusDefaultNetworkName,
-		CNIVersion:   cniVersion,
-		Type:         pluginName,
-		Capabilities: map[string]bool{},
-		Kubeconfig:   kubeconfig,
-		Delegates:    []interface{}{},
-	}
-
-	err := multusConfig.Mutate(configurationOptions...)
-	return multusConfig, err
-}
-
-// CheckVersionCompatibility checks compatibilty of the
-// top level cni version with the delegate cni version.
-// Since version 0.4.0, CHECK was introduced, which
-// causes incompatibility.
-func CheckVersionCompatibility(mc *MultusConf) error {
-	const versionFmt = "delegate cni version is %s while top level cni version is %s"
-	v040, _ := semver.Make("0.4.0")
-	multusCNIVersion, err := semver.Make(mc.CNIVersion)
-
-	if err != nil {
-		return errors.New("couldn't get top level cni version")
-	}
-
-	if multusCNIVersion.GTE(v040) {
-		for _, delegate := range mc.Delegates {
-			delegatesMap, ok := delegate.(map[string]interface{})
-			if !ok {
-				return errors.New("couldn't get cni version of delegate")
-			}
-			delegateVersion, ok := delegatesMap["cniVersion"].(string)
-			if !ok {
-				return errors.New("couldn't get cni version of delegate")
-			}
-			v, err := semver.Make(delegateVersion)
-			if err != nil {
-				return err
-			}
-			if v.LT(v040) {
-				return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
-			}
-		}
-	}
-
-	return nil
-}
-
-// Generate generates the multus configuration from whatever state is currently
-// held
-func (mc *MultusConf) Generate() (string, error) {
-	data, err := json.Marshal(mc)
-	return string(data), err
-}
-
-// Mutate updates the MultusConf attributes according to the provided
-// configuration `Option`s
-func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
-	for _, configOption := range configurationOptions {
-		configOption(mc)
-	}
-
-	return CheckVersionCompatibility(mc)
-}
-
-// WithNamespaceIsolation mutates the inner state to enable the
-// NamespaceIsolation attribute
-func WithNamespaceIsolation() Option {
-	return func(conf *MultusConf) {
-		conf.NamespaceIsolation = true
-	}
-}
-
-// WithGlobalNamespaces mutates the inner state to set the
-// RawNonIsolatedNamespaces attribute
-func WithGlobalNamespaces(globalNamespaces string) Option {
-	return func(conf *MultusConf) {
-		conf.RawNonIsolatedNamespaces = globalNamespaces
-	}
-}
-
-// WithLogToStdErr mutates the inner state to enable the
-// WithLogToStdErr attribute
-func WithLogToStdErr() Option {
-	return func(conf *MultusConf) {
-		conf.LogToStderr = true
-	}
-}
-
-// WithLogLevel mutates the inner state to set the
-// LogLevel attribute
-func WithLogLevel(logLevel string) Option {
-	return func(conf *MultusConf) {
-		conf.LogLevel = logLevel
-	}
-}
-
-// WithLogFile mutates the inner state to set the
-// logFile attribute
-func WithLogFile(logFile string) Option {
-	return func(conf *MultusConf) {
-		conf.LogFile = logFile
-	}
-}
-
-// WithLogOptions mutates the inner state to set the
-// LogOptions attribute
-func WithLogOptions(logOptions *LogOptions) Option {
-	return func(conf *MultusConf) {
-		conf.LogOptions = logOptions
-	}
-}
-
-// WithReadinessFileIndicator mutates the inner state to set the
-// ReadinessIndicatorFile attribute
-func WithReadinessFileIndicator(path string) Option {
-	return func(conf *MultusConf) {
-		conf.ReadinessIndicatorFile = path
-	}
-}
-
-// WithAdditionalBinaryFileDir mutates the inner state to set the
-// BinDir attribute
-func WithAdditionalBinaryFileDir(directoryPath string) Option {
-	return func(conf *MultusConf) {
-		conf.BinDir = directoryPath
-	}
-}
-
-// WithOverriddenName mutates the inner state to set the
-// Name attribute
-func WithOverriddenName(networkName string) Option {
-	return func(conf *MultusConf) {
-		conf.Name = networkName
-	}
-}
-
-func withCapabilities(cniData interface{}) Option {
-	var enabledCapabilities []string
-	var pluginsList []interface{}
-	cniDataMap, ok := cniData.(map[string]interface{})
-	if ok {
-		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
-			pluginsList = pluginsListEntry.([]interface{})
-		}
-	}
-
-	if len(pluginsList) > 0 {
-		for _, pluginData := range pluginsList {
-			enabledCapabilities = append(
-				enabledCapabilities,
-				extractCapabilities(pluginData)...)
-		}
-	} else {
-		enabledCapabilities = extractCapabilities(cniData)
-	}
-
-	return func(conf *MultusConf) {
-		for _, capability := range enabledCapabilities {
-			conf.Capabilities[capability] = true
-		}
-	}
-}
-
-func withDelegates(primaryCNIConfigData map[string]interface{}) Option {
-	return func(conf *MultusConf) {
-		conf.Delegates = []interface{}{primaryCNIConfigData}
-	}
-}
-
-// MutateLogOptions update the LoggingOptions of the MultusConf according
-// to the provided configuration `loggingOptions`
-func MutateLogOptions(logOption *LogOptions, logOptionFunc ...LogOptionFunc) {
-	for _, loggingOption := range logOptionFunc {
-		loggingOption(logOption)
-	}
-}
-
-// WithLogMaxSize mutates the inner state to set the
-// logMaxSize attribute
-func WithLogMaxSize(maxSize *int) LogOptionFunc {
-	return func(logOptions *LogOptions) {
-		logOptions.MaxSize = maxSize
-	}
-}
-
-// WithLogMaxAge mutates the inner state to set the
-// logMaxAge attribute
-func WithLogMaxAge(maxAge *int) LogOptionFunc {
-	return func(logOptions *LogOptions) {
-		logOptions.MaxAge = maxAge
-	}
-}
-
-// WithLogMaxBackups mutates the inner state to set the
-// logMaxBackups attribute
-func WithLogMaxBackups(maxBackups *int) LogOptionFunc {
-	return func(logOptions *LogOptions) {
-		logOptions.MaxBackups = maxBackups
-	}
-}
-
-// WithLogCompress mutates the inner state to set the
-// logCompress attribute
-func WithLogCompress(compress *bool) LogOptionFunc {
-	return func(logOptions *LogOptions) {
-		logOptions.Compress = compress
-	}
-}
-
-func extractCapabilities(capabilitiesInterface interface{}) []string {
-	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
-	if !ok {
-		return nil
-	}
-	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
-	if !ok {
-		return nil
-	}
-	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
-	if !ok {
-		return nil
-	}
-
-	var enabledCapabilities []string
-	if len(capabilities) > 0 {
-		for capName, isCapabilityEnabled := range capabilities {
-			if isCapabilityEnabled.(bool) {
-				enabledCapabilities = append(enabledCapabilities, capName)
-			}
-		}
-	}
-	return enabledCapabilities
-}
-
-func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
-	if remainingTries == 0 {
-		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
-	}
-	var cniPluginConfigs []string
-	files, err := ioutil.ReadDir(cniConfigDirPath)
-	if err != nil {
-		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
-	}
-
-	for _, file := range files {
-		if strings.HasPrefix(file.Name(), "00-multus") {
-			continue
-		}
-		fileExtension := filepath.Ext(file.Name())
-		if fileExtension == ".conf" || fileExtension == ".conflist" {
-			cniPluginConfigs = append(cniPluginConfigs, file.Name())
-		}
-	}
-
-	if len(cniPluginConfigs) == 0 {
-		time.Sleep(time.Second)
-		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
-	}
-	sort.Strings(cniPluginConfigs)
-	return cniPluginConfigs[0], nil
-}

pkg/config/generator_test.go

@@ -1,371 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package config
-
-import (
-	"encoding/json"
-	"fmt"
-	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
-	"testing"
-)
-
-const (
-	primaryCNIName = "myCNI"
-	cniVersion     = "0.4.0"
-	kubeconfig     = "/a/b/c/kubeconfig.kubeconfig"
-)
-
-type testCase struct {
-	t                        *testing.T
-	configGenerationFunction func() (string, error)
-}
-
-var primaryCNIConfig = map[string]interface{}{
-	"cniVersion":         "1.0.0",
-	"name":               "ovn-kubernetes",
-	"type":               "ovn-k8s-cni-overlay",
-	"ipam":               "{}",
-	"dns":                "{}",
-	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
-	"logLevel":           "5",
-	"logfile-maxsize":    100,
-	"logfile-maxbackups": 5,
-	"logfile-maxage":     5,
-}
-
-func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
-	multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
-	if err != nil {
-		return multusConfig, err
-	}
-	return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{})))
-}
-
-func TestBasicMultusConfig(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig)
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithNamespaceIsolation())
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithReadinessIndicator(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithReadinessFileIndicator("/a/b/u/it-lives"))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogLevel("notice"),
-		WithLogToStdErr(),
-		WithLogFile("/u/y/w/log.1"))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithLogOptionsConfiguration(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogOptions(&LogOptions{
-			MaxAge:     testutils.Int(5),
-			MaxSize:    testutils.Int(100),
-			MaxBackups: testutils.Int(5),
-			Compress:   testutils.Bool(true),
-		}))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5,\"maxSize\":100,\"maxBackups\":5,\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusLogOptionsWithLogMaxAge(t *testing.T) {
-	logOption := &LogOptions{}
-	MutateLogOptions(logOption, WithLogMaxAge(testutils.Int(5)))
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogOptions(logOption))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusLogOptionsWithLogMaxSize(t *testing.T) {
-	logOption := &LogOptions{}
-	MutateLogOptions(logOption, WithLogMaxSize(testutils.Int(100)))
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogOptions(logOption))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxSize\":100},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusLogOptionsWithLogBackups(t *testing.T) {
-	logOption := &LogOptions{}
-	MutateLogOptions(logOption, WithLogMaxBackups(testutils.Int(5)))
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogOptions(logOption))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxBackups\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusLogOptionsWithLogCompress(t *testing.T) {
-	logOption := &LogOptions{}
-	MutateLogOptions(logOption, WithLogCompress(testutils.Bool(true)))
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithLogOptions(logOption))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithGlobalNamespace(t *testing.T) {
-	const globalNamespace = "come-along-ns"
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithGlobalNamespaces(globalNamespace))
-	assertError(t, err, nil)
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
-	const anotherCNIBinDir = "a-dir-somewhere"
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithAdditionalBinaryFileDir(anotherCNIBinDir))
-	assertError(t, err, nil)
-	expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithCapabilities(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`{"capabilities": {"portMappings": true}}`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
-	multusConfig, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		withCapabilities(
-			documentHelper(`
-{
-    "plugins": [
-        {
-            "capabilities": {
-                "portMappings": true
-            }
-        },
-        {
-            "capabilities": {
-                "tuning": false
-            }
-        }
-    ]
-}`)))
-	assertError(t, err, nil)
-	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func assertError(t *testing.T, actual error, expected error) {
-	if actual != nil && expected != nil {
-		if actual.Error() != expected.Error() {
-			t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
-		}
-	}
-
-	if actual == nil && expected != nil {
-		t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
-	} else if actual != nil && expected == nil {
-		t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
-	}
-}
-
-func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
-	return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
-}
-
-func TestVersionIncompatibility(t *testing.T) {
-	const delegateCNIVersion = "0.3.0"
-
-	primaryCNIConfigOld := primaryCNIConfig
-	tmpVer := primaryCNIConfig["cniVersion"]
-	primaryCNIConfig["cniVersion"] = delegateCNIVersion
-	_, err := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfigOld)
-	primaryCNIConfig["cniVersion"] = tmpVer
-
-	assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
-}
-
-func TestMultusConfigWithOverriddenName(t *testing.T) {
-	newNetworkName := "mega-net-2000"
-	multusConfig, _ := newMultusConfigWithDelegates(
-		primaryCNIName,
-		cniVersion,
-		kubeconfig,
-		primaryCNIConfig,
-		WithOverriddenName(newNetworkName))
-	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
-	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
-}
-
-func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
-	return &testCase{
-		t:                        t,
-		configGenerationFunction: configGenerationFunc,
-	}
-}
-
-func (tc testCase) assertResult(expectedResult string) {
-	multusCNIConfig, err := tc.configGenerationFunction()
-	if err != nil {
-		tc.t.Fatalf("error generating multus configuration: %v", err)
-	}
-	if multusCNIConfig != expectedResult {
-		tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
-	}
-}
-
-func documentHelper(pluginInfo string) interface{} {
-	dp, _ := documentCNIData([]byte(pluginInfo))
-	return dp
-}
-
-func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
-	var cniData interface{}
-	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
-		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
-	}
-	return cniData, nil
-}

pkg/config/manager_test.go

@@ -1,133 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package config
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-const suiteName = "Configuration Manager"
-
-func TestMultusConfigurationManager(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, suiteName)
-}
-
-var _ = Describe(suiteName, func() {
-	const (
-		primaryCNIPluginName     = "00-mycni.conf"
-		primaryCNIPluginTemplate = `
-{
-  "cniVersion": "0.4.0",
-  "name": "mycni-name",
-  "type": "mycni",
-  "ipam": {},
-  "dns": {}
-}
-`
-	)
-
-	var configManager *Manager
-	var multusConfigDir string
-	var defaultCniConfig string
-
-	BeforeEach(func() {
-		var err error
-		multusConfigDir, err = ioutil.TempDir("", "multus-config")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
-	})
-
-	BeforeEach(func() {
-		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
-		Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), userRWPermission)).To(Succeed())
-
-		multusConf, _ := NewMultusConfig(
-			primaryCNIName,
-			cniVersion,
-			kubeconfig)
-		var err error
-		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName)
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
-	})
-
-	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
-		expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
-		config, err := configManager.GenerateConfig()
-		Expect(err).NotTo(HaveOccurred())
-		Expect(config).To(Equal(expectedResult))
-	})
-
-	Context("Updates to the delegate CNI configuration", func() {
-		var (
-			doneChannel chan struct{}
-			stopChannel chan struct{}
-		)
-
-		BeforeEach(func() {
-			doneChannel = make(chan struct{})
-			stopChannel = make(chan struct{})
-			go func() {
-				Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
-			}()
-		})
-
-		AfterEach(func() {
-			go func() { stopChannel <- struct{}{} }()
-			Eventually(<-doneChannel).Should(Equal(struct{}{}))
-			close(doneChannel)
-			close(stopChannel)
-		})
-
-		It("Trigger the re-generation of the Multus CNI configuration", func() {
-			newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
-			Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), userRWPermission)).To(Succeed())
-
-			multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
-			Eventually(func() (string, error) {
-				multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
-				return string(multusCniData), err
-			}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
-		})
-	})
-
-	When("the user requests the name of the multus configuration to be overridden", func() {
-		BeforeEach(func() {
-			Expect(configManager.OverrideNetworkName()).To(Succeed())
-		})
-
-		It("Overrides the name of the multus configuration when requested", func() {
-			expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
-			config, err := configManager.GenerateConfig()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(config).To(Equal(expectedResult))
-		})
-	})
-})
-
-func multusConfigFromDelegate(delegateConfig string) string {
-	return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
-}

pkg/k8sclient/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8sclient/k8sclient.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -41,9 +42,9 @@ import (
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/kubeletclient"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/kubeletclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 )
 
 const (
@@ -96,8 +97,17 @@ func (e *NoK8sNetworkError) Error() string { return e.message }
 
 // SetNetworkStatus sets network status into Pod annotation
 func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
+	podName := string(k8sArgs.K8S_POD_NAME)
+	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podUID := string(k8sArgs.K8S_POD_UID)
+
+	return SetPodNetworkStatusAnnotation(client, podName, podNamespace, podUID, netStatus, conf)
+}
+
+// SetPodNetworkStatusAnnotation sets network status into Pod annotation
+func SetPodNetworkStatusAnnotation(client *ClientInfo, podName string, podNamespace string, podUID string, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
 	var err error
-	logging.Debugf("SetNetworkStatus: %v, %v, %v, %v", client, k8sArgs, netStatus, conf)
+	logging.Debugf("SetPodNetworkStatusAnnotation: %v, %v, %v", client, netStatus, conf)
 
 	client, err = GetK8sClient(conf.Kubeconfig, client)
 	if err != nil {
@@ -108,16 +118,13 @@ func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []ne
 			// No available kube client and no delegates, we can't do anything
 			return logging.Errorf("SetNetworkStatus: must have either Kubernetes config or delegates")
 		}
-		logging.Debugf("SetNetworkStatus: kube client info is not defined, skip network status setup")
+		logging.Debugf("SetPodNetworkStatusAnnotation: kube client info is not defined, skip network status setup")
 		return nil
 	}
 
-	podName := string(k8sArgs.K8S_POD_NAME)
-	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
-	podUID := string(k8sArgs.K8S_POD_UID)
 	pod, err := client.GetPod(podNamespace, podName)
 	if err != nil {
-		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", podName, err)
+		return logging.Errorf("SetPodNetworkStatusAnnotation: failed to query the pod %v in out of cluster comm: %v", podName, err)
 	}
 
 	if podUID != "" && string(pod.UID) != podUID && !IsStaticPod(pod) {
@@ -127,7 +134,7 @@ func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []ne
 	if netStatus != nil {
 		err = netutils.SetNetworkStatus(client.Client, pod, netStatus)
 		if err != nil {
-			return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", podName, err)
+			return logging.Errorf("SetPodNetworkStatusAnnotation: failed to update the pod %v in out of cluster comm: %v", podName, err)
 		}
 	}
 
@@ -183,7 +190,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 		return nil, logging.Errorf("parsePodNetworkAnnotation: pod annotation does not have \"network\" as key")
 	}
 
-	if strings.IndexAny(podNetworks, "[{\"") >= 0 {
+	if strings.ContainsAny(podNetworks, "[{\"") {
 		if err := json.Unmarshal([]byte(podNetworks), &networks); err != nil {
 			return nil, logging.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err)
 		}
@@ -285,6 +292,11 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		}
 	}
 
+	// acquire lock to access file
+	if types.ChrootMutex != nil {
+		types.ChrootMutex.Lock()
+		defer types.ChrootMutex.Unlock()
+	}
 	configBytes, err := netutils.GetCNIConfig(customResource, confdir)
 	if err != nil {
 		return nil, resourceMap, err
@@ -365,7 +377,7 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 			}
 		}
 
-		if isGatewayConfigured == true {
+		if isGatewayConfigured {
 			err = types.CheckGatewayConfig(conf.Delegates)
 			if err != nil {
 				return 0, nil, err
@@ -381,6 +393,18 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 	return 0, clientInfo, err
 }
 
+// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
+// the k8s API.
+func InClusterK8sClient() (*ClientInfo, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	logging.Debugf("InClusterK8sClient: in cluster config: %+v", config)
+	return NewClientInfo(config)
+}
+
 // GetK8sClient gets client info from kubeconfig
 func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
 	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
@@ -417,7 +441,12 @@ func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error
 	// Set the config timeout to one minute.
 	config.Timeout = time.Minute
 
-	// creates the clientset
+	return NewClientInfo(config)
+}
+
+// NewClientInfo returns a `ClientInfo` from a configuration created from an
+// existing kubeconfig file.
+func NewClientInfo(config *rest.Config) (*ClientInfo, error) {
 	client, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return nil, err
@@ -432,7 +461,6 @@ func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error
 	broadcaster.StartLogging(klog.Infof)
 	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
 	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
-
 	return &ClientInfo{
 		Client:           client,
 		NetClient:        netclient,
@@ -500,32 +528,53 @@ func isValidNamespaceReference(targetns string, allowednamespaces []string) bool
 	return false
 }
 
+// getNetDelegate loads delegate network for clusterNetwork/defaultNetworks
 func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 	logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace)
-	// option1) search CRD object for the network
-	net := &types.NetworkSelectionElement{
-		Name:      netname,
-		Namespace: namespace,
-	}
-	delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
-	if err == nil {
-		return delegate, resourceMap, nil
-	}
-
-	// option2) search CNI json config file
 	var configBytes []byte
-	configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
-	if err == nil {
-		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
+	isNetnamePath := strings.Contains(netname, "/")
+
+	// if netname is not directory or file, it must be net-attach-def name or CNI config name
+	if !isNetnamePath {
+		// option1) search CRD object for the network
+		net := &types.NetworkSelectionElement{
+			Name:      netname,
+			Namespace: namespace,
+		}
+		delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
+		if err == nil {
+			return delegate, resourceMap, nil
+		}
+
+		// option2) search CNI json config file, which has <netname> as CNI name, from confDir
+
+		// acquire lock to access file
+		if types.ChrootMutex != nil {
+			types.ChrootMutex.Lock()
+			defer types.ChrootMutex.Unlock()
+		}
+
+		configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
+		if err == nil {
+			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
+			if err != nil {
+				return nil, resourceMap, err
+			}
+			return delegate, resourceMap, nil
+		}
+	} else {
+		// acquire lock to access file
+		if types.ChrootMutex != nil {
+			types.ChrootMutex.Lock()
+			defer types.ChrootMutex.Unlock()
+		}
+
+		fInfo, err := os.Stat(netname)
 		if err != nil {
 			return nil, resourceMap, err
 		}
-		return delegate, resourceMap, nil
-	}
 
-	// option3) search directory
-	fInfo, err := os.Stat(netname)
-	if err == nil {
+		// option3) search directory
 		if fInfo.IsDir() {
 			files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
 			if err != nil {
@@ -543,6 +592,29 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 				}
 				return nil, resourceMap, err
 			}
+		} else {
+			// option4) if file path (absolute), then load it directly
+			if strings.HasSuffix(netname, ".conflist") {
+				confList, err := libcni.ConfListFromFile(netname)
+				if err != nil {
+					return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
+				}
+				configBytes = confList.Bytes
+			} else {
+				conf, err := libcni.ConfFromFile(netname)
+				if err != nil {
+					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
+				}
+				if conf.Network.Type == "" {
+					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
+				}
+				configBytes = conf.Bytes
+			}
+			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
+			if err != nil {
+				return nil, resourceMap, err
+			}
+			return delegate, resourceMap, nil
 		}
 	}
 	return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
@@ -626,7 +698,7 @@ const ConfigSourceAnnotationKey = "kubernetes.io/config.source"
 // IsStaticPod returns true if the pod is static pod.
 func IsStaticPod(pod *v1.Pod) bool {
 	if pod.Annotations != nil {
-		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok == true {
+		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok {
 			return source != "api"
 		}
 	}

pkg/k8sclient/k8sclient_test.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,22 +12,20 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 package k8sclient
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 
 	types020 "github.com/containernetworking/cni/pkg/types/020"
-	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
@@ -34,7 +33,7 @@ import (
 
 	"k8s.io/client-go/kubernetes/fake"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
@@ -60,7 +59,7 @@ var _ = Describe("k8sclient operations", func() {
 	const fakePodName string = "testPod"
 
 	BeforeEach(func() {
-		tmpDir, err = ioutil.TempDir("", "multus_tmp")
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
 		Expect(err).NotTo(HaveOccurred())
 		genericConf = `{
 			"name":"node-cni-network",
@@ -512,7 +511,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
 	})
 
-	It("retrieves cluster network from path", func() {
+	It("retrieves cluster network from directory path", func() {
 		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
@@ -544,6 +543,37 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
 	})
 
+	It("retrieves cluster network from cni config path", func() {
+		net1Name := filepath.Join(tmpDir, "10-net1.conf")
+		os.WriteFile(net1Name, []byte(`{
+				"name": "net1",
+				"type": "mynet",
+				"cniVersion": "0.3.1"
+			}`), 0600)
+
+		fakePod := testutils.NewFakePod(fakePodName, "", "")
+		conf := fmt.Sprintf(`{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "%s",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`, net1Name)
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		clientInfo := NewFakeClientInfo()
+		_, err = clientInfo.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+	})
+
 	It("Error in case of CRD not found", func() {
 		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
@@ -734,7 +764,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("uses cached delegates when an error in loading from pod annotation occurs", func() {
-		dir, err := ioutil.TempDir("", "multus-test")
+		dir, err := os.MkdirTemp("", "multus-test")
 		Expect(err).NotTo(HaveOccurred())
 		defer os.RemoveAll(dir) // clean up
 

pkg/kubeletclient/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/kubeletclient/kubeletclient.go

@@ -1,49 +1,98 @@
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubeletclient
 
 import (
+	"fmt"
+	"net"
 	"net/url"
 	"os"
+	"path/filepath"
 	"time"
 
 	"golang.org/x/net/context"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/checkpoint"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/checkpoint"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
-	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
-	"k8s.io/kubernetes/pkg/kubelet/util"
 )
 
 const (
-	defaultKubeletSocketFile   = "kubelet.sock"
+	defaultKubeletSocket       = "kubelet" // which is defined in k8s.io/kubernetes/pkg/kubelet/apis/podresources
+	kubeletConnectionTimeout   = 10 * time.Second
 	defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
 	defaultPodResourcesPath    = "/var/lib/kubelet/pod-resources"
+	unixProtocol               = "unix"
 )
 
+// LocalEndpoint returns the full path to a unix socket at the given endpoint
+// which is in k8s.io/kubernetes/pkg/kubelet/util
+func localEndpoint(path string) *url.URL {
+	return &url.URL{
+		Scheme: unixProtocol,
+		Path:   path + ".sock",
+	}
+}
+
 // GetResourceClient returns an instance of ResourceClient interface initialized with Pod resource information
 func GetResourceClient(kubeletSocket string) (types.ResourceClient, error) {
-	if kubeletSocket == "" {
-		kubeletSocket, _ = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
+	kubeletSocketURL := localEndpoint(filepath.Join(defaultPodResourcesPath, defaultKubeletSocket))
+
+	if kubeletSocket != "" {
+		kubeletSocketURL = &url.URL{
+			Scheme: unixProtocol,
+			Path:   kubeletSocket,
+		}
 	}
 	// If Kubelet resource API endpoint exist use that by default
 	// Or else fallback with checkpoint file
-	if hasKubeletAPIEndpoint(kubeletSocket) {
+	if hasKubeletAPIEndpoint(kubeletSocketURL) {
 		logging.Debugf("GetResourceClient: using Kubelet resource API endpoint")
-		return getKubeletClient(kubeletSocket)
+		return getKubeletClient(kubeletSocketURL)
 	}
 
 	logging.Debugf("GetResourceClient: using Kubelet device plugin checkpoint")
 	return checkpoint.GetCheckpoint()
 }
 
-func getKubeletClient(kubeletSocket string) (types.ResourceClient, error) {
-	newClient := &kubeletClient{}
-	if kubeletSocket == "" {
-		kubeletSocket, _ = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
-	}
+func dial(ctx context.Context, addr string) (net.Conn, error) {
+	return (&net.Dialer{}).DialContext(ctx, unixProtocol, addr)
+}
 
-	client, conn, err := podresources.GetV1Client(kubeletSocket, 10*time.Second, defaultPodResourcesMaxSize)
+func getKubeletResourceClient(kubeletSocketURL *url.URL, timeout time.Duration) (podresourcesapi.PodResourcesListerClient, *grpc.ClientConn, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, kubeletSocketURL.Path, grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithContextDialer(dial),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaultPodResourcesMaxSize)))
+	if err != nil {
+		return nil, nil, fmt.Errorf("error dialing socket %s: %v", kubeletSocketURL.Path, err)
+	}
+	return podresourcesapi.NewPodResourcesListerClient(conn), conn, nil
+}
+
+func getKubeletClient(kubeletSocketURL *url.URL) (types.ResourceClient, error) {
+	newClient := &kubeletClient{}
+
+	client, conn, err := getKubeletResourceClient(kubeletSocketURL, 10*time.Second)
 	if err != nil {
 		return nil, logging.Errorf("getKubeletClient: error getting grpc client: %v\n", err)
 	}
@@ -82,7 +131,7 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou
 	ns := pod.Namespace
 
 	if name == "" || ns == "" {
-		return nil, logging.Errorf("GetPodResourcesMap: Pod name or namespace cannot be empty")
+		return nil, logging.Errorf("GetPodResourceMap: Pod name or namespace cannot be empty")
 	}
 
 	for _, pr := range rc.resources {
@@ -101,13 +150,9 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou
 	return resourceMap, nil
 }
 
-func hasKubeletAPIEndpoint(endpoint string) bool {
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return false
-	}
+func hasKubeletAPIEndpoint(url *url.URL) bool {
 	// Check for kubelet resource API socket file
-	if _, err := os.Stat(u.Path); err != nil {
+	if _, err := os.Stat(url.Path); err != nil {
 		logging.Debugf("hasKubeletAPIEndpoint: error looking up kubelet resource api socket file: %q", err)
 		return false
 	}

pkg/kubeletclient/kubeletclient_test.go

@@ -1,21 +1,39 @@
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubeletclient
 
 import (
 	"context"
-	"io/ioutil"
+	"fmt"
+	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
-	"k8s.io/kubernetes/pkg/kubelet/util"
 
-	mtypes "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	mtypes "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
 )
 
@@ -29,14 +47,12 @@ type fakeResourceServer struct {
 	server *grpc.Server
 }
 
-/* This is for 1.21.x or later. Uncomment it once we update vendor here!
-//TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
-func (m *fakeResourceServer) GetAllocatableResources(ctx context.Context, req *podresourcesapi.AllocatableResourcesRequest) (*podresourcesapi.AllocatableResourcesResponse, error) {
+// TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
+func (m *fakeResourceServer) GetAllocatableResources(_ context.Context, _ *podresourcesapi.AllocatableResourcesRequest) (*podresourcesapi.AllocatableResourcesResponse, error) {
 	return &podresourcesapi.AllocatableResourcesResponse{}, nil
 }
-*/
 
-func (m *fakeResourceServer) List(ctx context.Context, req *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
+func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
 	podName := "pod-name"
 	podNamespace := "pod-namespace"
 	containerName := "container-name"
@@ -70,10 +86,45 @@ func TestKubeletclient(t *testing.T) {
 	RunSpecs(t, "Kubeletclient Suite")
 }
 
-var testKubeletSocket string
+var testKubeletSocket *url.URL
+
+// CreateListener creates a listener on the specified endpoint.
+// based from k8s.io/kubernetes/pkg/kubelet/util
+func CreateListener(addr string) (net.Listener, error) {
+	// Unlink to cleanup the previous socket file.
+	err := unix.Unlink(addr)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("failed to unlink socket file %q: %v", addr, err)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(addr), 0750); err != nil {
+		return nil, fmt.Errorf("error creating socket directory %q: %v", filepath.Dir(addr), err)
+	}
+
+	// Create the socket on a tempfile and move it to the destination socket to handle improper cleanup
+	file, err := os.CreateTemp(filepath.Dir(addr), "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file: %v", err)
+	}
+
+	if err := os.Remove(file.Name()); err != nil {
+		return nil, fmt.Errorf("failed to remove temporary file: %v", err)
+	}
+
+	l, err := net.Listen(unixProtocol, file.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	if err = os.Rename(file.Name(), addr); err != nil {
+		return nil, fmt.Errorf("failed to move temporary file to addr %q: %v", addr, err)
+	}
+
+	return l, nil
+}
 
 func setUp() error {
-	tempSocketDir, err := ioutil.TempDir("", "kubelet-resource-client")
+	tempSocketDir, err := os.MkdirTemp("", "kubelet-resource-client")
 	if err != nil {
 		return err
 	}
@@ -85,11 +136,11 @@ func setUp() error {
 
 	socketDir = testingPodResourcesPath
 	socketName = filepath.Join(socketDir, "kubelet.sock")
-	testKubeletSocket = socketName
+	testKubeletSocket = localEndpoint(filepath.Join(socketDir, "kubelet"))
 
 	fakeServer = &fakeResourceServer{server: grpc.NewServer()}
 	podresourcesapi.RegisterPodResourcesListerServer(fakeServer.server, fakeServer)
-	lis, err := util.CreateListener(socketName)
+	lis, err := CreateListener(socketName)
 	if err != nil {
 		return err
 	}
@@ -119,16 +170,16 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 
 	Context("GetResourceClient()", func() {
 		It("should return no error", func() {
-			_, err := GetResourceClient(testKubeletSocket)
+			_, err := GetResourceClient(testKubeletSocket.Path)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("should fail with missing file", func() {
-			_, err := GetResourceClient("sampleSocketString")
+			_, err := GetResourceClient("unix:/sampleSocketString")
 			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("error reading file"))
 		})
 	})
-
 	Context("GetPodResourceMap() with valid pod name and namespace", func() {
 		It("should return no error", func() {
 			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
@@ -159,7 +210,9 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 		})
 
 		It("should return an error with garbage socket value", func() {
-			_, err := getKubeletClient("/badfilepath!?//")
+			u, err := url.Parse("/badfilepath!?//")
+			Expect(err).NotTo(HaveOccurred())
+			_, err = getKubeletClient(u)
 			Expect(err).To(HaveOccurred())
 		})
 	})

pkg/logging/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/logging/logging.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,13 +16,13 @@
 package logging
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/pkg/errors"
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
 

pkg/logging/logging_test.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,11 +16,14 @@
 package logging
 
 import (
-	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
-	"gopkg.in/natefinch/lumberjack.v2"
+	"fmt"
+	"os"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
+	"gopkg.in/natefinch/lumberjack.v2"
+
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
@@ -56,12 +60,16 @@ var _ = Describe("logging operations", func() {
 	It("Check loglevel setter", func() {
 		SetLogLevel("debug")
 		Expect(loggingLevel).To(Equal(DebugLevel))
+		Expect(loggingLevel.String()).To(Equal("debug"))
 		SetLogLevel("Error")
 		Expect(loggingLevel).To(Equal(ErrorLevel))
+		Expect(loggingLevel.String()).To(Equal("error"))
 		SetLogLevel("VERbose")
 		Expect(loggingLevel).To(Equal(VerboseLevel))
+		Expect(loggingLevel.String()).To(Equal("verbose"))
 		SetLogLevel("PANIC")
 		Expect(loggingLevel).To(Equal(PanicLevel))
+		Expect(loggingLevel.String()).To(Equal("panic"))
 	})
 
 	It("Check loglevel setter with invalid level", func() {
@@ -76,6 +84,37 @@ var _ = Describe("logging operations", func() {
 		Expect(loggingStderr).NotTo(Equal(currentVal))
 	})
 
+	It("Check log function is worked", func() {
+		Debugf("foobar")
+		Verbosef("foobar")
+		Expect(Errorf("foobar")).NotTo(BeNil())
+		Panicf("foobar")
+	})
+
+	It("Check log function is worked with stderr", func() {
+		SetLogStderr(true)
+		Debugf("foobar")
+		Verbosef("foobar")
+		Expect(Errorf("foobar")).NotTo(BeNil())
+		Panicf("foobar")
+	})
+
+	It("Check log function is worked with stderr", func() {
+		tmpDir, err := os.MkdirTemp("", "multus_tmp")
+		SetLogFile(fmt.Sprintf("%s/log.txt", tmpDir))
+		Debugf("foobar")
+		Verbosef("foobar")
+		Expect(Errorf("foobar")).NotTo(BeNil())
+		Panicf("foobar")
+		logger.Filename = ""
+		loggingW = nil
+		err = os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+		// Revert the log variable to init
+		loggingW = nil
+		logger = &lumberjack.Logger{}
+	})
+
 	// Tests public getter
 	It("Check getter for logging level with current level", func() {
 		currentLevel := loggingLevel

pkg/multus/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/multus/multus.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2016 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -29,7 +29,7 @@ import (
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
-	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
@@ -39,10 +39,10 @@ import (
 	k8snet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 
-	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/k8sclient"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/netutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/netutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 )
 
 const (
@@ -51,9 +51,11 @@ const (
 )
 
 var (
-	version = "master@git"
-	commit  = "unknown commit"
-	date    = "unknown date"
+	version       = "master@git"
+	commit        = "unknown commit"
+	date          = "unknown date"
+	gitTreeState  = ""
+	releaseStatus = ""
 )
 
 var (
@@ -63,8 +65,7 @@ var (
 
 // PrintVersionString ...
 func PrintVersionString() string {
-	return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s",
-		version, commit, date)
+	return fmt.Sprintf("version:%s(%s%s), commit:%s, date:%s", version, gitTreeState, releaseStatus, commit, date)
 }
 
 func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
@@ -75,7 +76,7 @@ func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
 
 	path := filepath.Join(dataDir, containerID)
 
-	err := ioutil.WriteFile(path, netconf, 0600)
+	err := os.WriteFile(path, netconf, 0600)
 	if err != nil {
 		return logging.Errorf("saveScratchNetConf: failed to write container data in the path(%q): %v", path, err)
 	}
@@ -87,7 +88,7 @@ func consumeScratchNetConf(containerID, dataDir string) ([]byte, string, error)
 	logging.Debugf("consumeScratchNetConf: %s, %s", containerID, dataDir)
 	path := filepath.Join(dataDir, containerID)
 
-	b, err := ioutil.ReadFile(path)
+	b, err := os.ReadFile(path)
 	return b, path, err
 }
 
@@ -106,7 +107,7 @@ func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string {
 	return fmt.Sprintf("net%d", idx)
 }
 
-func getDelegateDeviceInfo(delegate *types.DelegateNetConf, runtimeConf *libcni.RuntimeConf) (*nettypes.DeviceInfo, error) {
+func getDelegateDeviceInfo(_ *types.DelegateNetConf, runtimeConf *libcni.RuntimeConf) (*nettypes.DeviceInfo, error) {
 	// If the DPDeviceInfoFile was created, it was copied to the CNIDeviceInfoFile.
 	// If the DPDeviceInfoFile was not created, CNI might have created it. So
 	// either way, load CNIDeviceInfoFile.
@@ -286,11 +287,12 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *t
 	return err
 }
 
-func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
-	logging.Debugf("delegateAdd: %v, %v, %v", exec, delegate, rt)
+// DelegateAdd ...
+func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
+	logging.Debugf("DelegateAdd: %v, %v, %v", exec, delegate, rt)
 
 	if err := validateIfName(rt.NetNS, rt.IfName); err != nil {
-		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, rt.IfName, err)
+		return nil, logging.Errorf("DelegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, rt.IfName, err)
 	}
 
 	// Deprecated in ver 3.5.
@@ -299,10 +301,10 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 			// validate Mac address
 			_, err := net.ParseMAC(delegate.MacRequest)
 			if err != nil {
-				return nil, logging.Errorf("delegateAdd: failed to parse mac address %q", delegate.MacRequest)
+				return nil, logging.Errorf("DelegateAdd: failed to parse mac address %q", delegate.MacRequest)
 			}
 
-			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, rt.IfName)
+			logging.Debugf("DelegateAdd: set MAC address %q to %q", delegate.MacRequest, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"MAC", delegate.MacRequest})
 		}
 
@@ -312,15 +314,15 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 				if strings.Contains(ip, "/") {
 					_, _, err := net.ParseCIDR(ip)
 					if err != nil {
-						return nil, logging.Errorf("delegateAdd: failed to parse IP address %q", ip)
+						return nil, logging.Errorf("DelegateAdd: failed to parse IP address %q", ip)
 					}
 				} else if net.ParseIP(ip) == nil {
-					return nil, logging.Errorf("delegateAdd: failed to parse IP address %q", ip)
+					return nil, logging.Errorf("DelegateAdd: failed to parse IP address %q", ip)
 				}
 			}
 
 			ips := strings.Join(delegate.IPRequest, ",")
-			logging.Debugf("delegateAdd: set IP address %q to %q", ips, rt.IfName)
+			logging.Debugf("DelegateAdd: set IP address %q to %q", ips, rt.IfName)
 			rt.Args = append(rt.Args, [2]string{"IP", ips})
 		}
 	}
@@ -357,9 +359,9 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 
 	// get IP addresses from result
 	ips := []string{}
-	res, err := cnicurrent.NewResultFromResult(result)
+	res, err := cni100.NewResultFromResult(result)
 	if err != nil {
-		logging.Errorf("delegateAdd: error converting result: %v", err)
+		logging.Errorf("DelegateAdd: error converting result: %v", err)
 		return result, nil
 	}
 	for _, ip := range res.IPs {
@@ -375,14 +377,14 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 		}
 	} else {
 		// for further debug https://github.com/k8snetworkplumbingwg/multus-cni/issues/481
-		logging.Errorf("delegateAdd: pod nil pointer: namespace: %s, name: %s, container id: %s, pod: %v", rt.Args[1][1], rt.Args[2][1], rt.Args[3][1], pod)
+		logging.Errorf("DelegateAdd: pod nil pointer: namespace: %s, name: %s, container id: %s, pod: %v", rt.Args[1][1], rt.Args[2][1], rt.Args[3][1], pod)
 	}
-
 	return result, nil
 }
 
-func delegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateCheck: %v, %v, %v", exec, delegateConf, rt)
+// DelegateCheck ...
+func DelegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("DelegateCheck: %v, %v, %v", exec, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var cniConfName string
@@ -398,20 +400,21 @@ func delegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *li
 	if delegateConf.ConfListPlugin {
 		err = conflistCheck(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
-			return logging.Errorf("delegateCheck: error invoking ConflistCheck - %q: %v", delegateConf.ConfList.Name, err)
+			return logging.Errorf("DelegateCheck: error invoking ConflistCheck - %q: %v", delegateConf.ConfList.Name, err)
 		}
 	} else {
 		err = confCheck(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
-			return logging.Errorf("delegateCheck: error invoking DelegateCheck - %q: %v", delegateConf.Conf.Type, err)
+			return logging.Errorf("DelegateCheck: error invoking DelegateCheck - %q: %v", delegateConf.Conf.Type, err)
 		}
 	}
 
 	return err
 }
 
-func delegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
+// DelegateDel ...
+func DelegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("DelegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var confName string
@@ -431,12 +434,12 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetC
 	if delegateConf.ConfListPlugin {
 		err = conflistDel(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
-			return logging.Errorf("delegateDel: error invoking ConflistDel - %q: %v", delegateConf.ConfList.Name, err)
+			return logging.Errorf("DelegateDel: error invoking ConflistDel - %q: %v", delegateConf.ConfList.Name, err)
 		}
 	} else {
 		err = confDel(rt, delegateConf.Bytes, multusNetconf, exec)
 		if err != nil {
-			return logging.Errorf("delegateDel: error invoking DelegateDel - %q: %v", delegateConf.Conf.Type, err)
+			return logging.Errorf("DelegateDel: error invoking DelegateDel - %q: %v", delegateConf.Conf.Type, err)
 		}
 	}
 
@@ -454,7 +457,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 		ifName := getIfname(delegates[idx], args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
-		if err := delegateDel(exec, pod, delegates[idx], rt, multusNetconf); err != nil {
+		if err := DelegateDel(exec, pod, delegates[idx], rt, multusNetconf); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
 		if cniDeviceInfoPath != "" {
@@ -503,7 +506,9 @@ func isCriticalRequestRetriable(err error) bool {
 	return false
 }
 
-func getPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
+// GetPod retrieves Kubernetes Pod object from given namespace/name in k8sArgs (i.e. cni args)
+// GetPod also get pod UID, but it is not used to retrieve, but it is used for double check
+func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
 	if kubeClient == nil {
 		return nil, nil
 	}
@@ -576,7 +581,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 	}
 
-	pod, err := getPod(kubeClient, k8sArgs, false)
+	pod, err := GetPod(kubeClient, k8sArgs, false)
 	if err != nil {
 		return nil, err
 	}
@@ -614,7 +619,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			// Even if the filename is set, file may not be present. Ignore error,
 			// but log and in the future may need to filter on specific errors.
 			if err != nil {
-				logging.Debugf("cmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
+				logging.Debugf("CmdAdd: CopyDeviceInfoForCNIFromDP returned an error - err=%v", err)
 			}
 		}
 
@@ -623,7 +628,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		if netName == "" {
 			netName = delegate.ConfList.Name
 		}
-		tmpResult, err = delegateAdd(exec, kubeClient, pod, delegate, rt, n)
+		tmpResult, err = DelegateAdd(exec, kubeClient, pod, delegate, rt, n)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
 			// Ignore errors; DEL must be idempotent anyway
@@ -669,7 +674,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 		// Remove gateway if `default-route` network selection is specified
 		if deleteV4gateway || deleteV6gateway {
-			err = netutils.DeleteDefaultGW(args, ifName)
+			err = netutils.DeleteDefaultGW(args.Netns, ifName)
 			if err != nil {
 				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
 			}
@@ -681,7 +686,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 		// Here we'll set the default gateway which specified in `default-route` network selection
 		if adddefaultgateway {
-			err = netutils.SetDefaultGW(args, ifName, *delegate.GatewayRequest)
+			err = netutils.SetDefaultGW(args.Netns, ifName, *delegate.GatewayRequest)
 			if err != nil {
 				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
 			}
@@ -702,11 +707,11 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		if err != nil {
 			// Even if the filename is set, file may not be present. Ignore error,
 			// but log and in the future may need to filter on specific errors.
-			logging.Debugf("cmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
+			logging.Debugf("CmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
 		}
 
 		// create the network status, only in case Multus as kubeconfig
-		if n.Kubeconfig != "" && kc != nil {
+		if kubeClient != nil && kc != nil {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
 				delegateNetStatus, err := nadutils.CreateNetworkStatus(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
 				if err != nil {
@@ -722,7 +727,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	}
 
 	// set the network status annotation in apiserver, only in case Multus as kubeconfig
-	if n.Kubeconfig != "" && kc != nil {
+	if kubeClient != nil && kc != nil {
 		if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
 			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
@@ -754,7 +759,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 		ifName := getIfname(delegate, args.IfName, idx)
 
 		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
-		err = delegateCheck(exec, delegate, rt, in)
+		err = DelegateCheck(exec, delegate, rt, in)
 		if err != nil {
 			return err
 		}
@@ -810,17 +815,35 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return cmdErr(nil, "error getting k8s client: %v", err)
 	}
 
-	pod, err := getPod(kubeClient, k8sArgs, true)
+	pod, err := GetPod(kubeClient, k8sArgs, true)
 	if err != nil {
-		// getPod may be failed but just do print error in its log and continue to delete
-		logging.Errorf("Multus: getPod failed: %v, but continue to delete", err)
+		// GetPod may be failed but just do print error in its log and continue to delete
+		logging.Errorf("Multus: GetPod failed: %v, but continue to delete", err)
 		// skip status update because k8s api seems to be stucked
 		skipStatusUpdate = true
 	}
 
 	// Read the cache to get delegates json for the pod
 	netconfBytes, path, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
-	if err != nil {
+	useCacheConf := false
+	if err == nil {
+		in.Delegates = []*types.DelegateNetConf{}
+		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
+			logging.Errorf("Multus: failed to load netconf: %v", err)
+		} else {
+			useCacheConf = true
+			// check plugins field and enable ConfListPlugin if there is
+			for _, v := range in.Delegates {
+				if len(v.ConfList.Plugins) != 0 {
+					v.ConfListPlugin = true
+				}
+			}
+			// First delegate is always the master plugin
+			in.Delegates[0].MasterPlugin = true
+		}
+	}
+
+	if !useCacheConf {
 		// Fetch delegates again if cache is not exist and pod info can be read
 		if os.IsNotExist(err) && pod != nil {
 			if in.ClusterNetwork != "" {
@@ -848,25 +871,11 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 			logging.Errorf("Multus: failed to get the cached delegates file: %v, cannot properly delete", err)
 			return nil
 		}
-	} else {
-		defer os.Remove(path)
-		in.Delegates = []*types.DelegateNetConf{}
-		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
-			return cmdErr(k8sArgs, "failed to load netconf: %v", err)
-		}
-		// check plugins field and enable ConfListPlugin if there is
-		for _, v := range in.Delegates {
-			if len(v.ConfList.Plugins) != 0 {
-				v.ConfListPlugin = true
-			}
-		}
-		// First delegate is always the master plugin
-		in.Delegates[0].MasterPlugin = true
 	}
 
 	// set CNIVersion in delegate CNI config if there is no CNIVersion and multus conf have CNIVersion.
 	for _, v := range in.Delegates {
-		if v.ConfListPlugin == true && v.ConfList.CNIVersion == "" && in.CNIVersion != "" {
+		if v.ConfListPlugin && v.ConfList.CNIVersion == "" && in.CNIVersion != "" {
 			v.ConfList.CNIVersion = in.CNIVersion
 			v.Bytes, err = json.Marshal(v.ConfList)
 			if err != nil {
@@ -877,7 +886,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 	}
 
 	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
-	if in.Kubeconfig != "" {
+	if kubeClient != nil {
 		if !skipStatusUpdate {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
 				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
@@ -891,5 +900,27 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
+	e := delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
+
+	// Enable Option only delegate plugin delete success to delete cache file
+	// CNI Runtime maybe return an error to block sandbox cleanup a while initiative,
+	// like starting, prepare something, it will be OK when retry later
+	// put "delete cache file" off later ensure have enough info delegate DEL message when Pod has been fully
+	// deleted from ETCD before sandbox cleanup success..
+	if in.RetryDeleteOnError {
+		if useCacheConf {
+			// Kubelet though this error as has been cleanup success and never retry, clean cache also
+			// Block sandbox cleanup error message can not contain "no such file or directory", CNI Runtime maybe should adaptor it !
+			if e == nil || strings.Contains(e.Error(), "no such file or directory") {
+				_ = os.Remove(path) // lgtm[go/path-injection]
+			}
+		}
+	} else {
+		if useCacheConf {
+			// remove used cache file
+			_ = os.Remove(path) // lgtm[go/path-injection]
+		}
+	}
+
+	return e
 }

pkg/multus/multus_cni020_test.go

@@ -0,0 +1,771 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multus
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/testutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("multus operations", func() {
+	It("fails to save NetConf with bad filepath", func() {
+		meme := []byte(`meme`)
+		err := saveScratchNetConf("123456789", "", meme)
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("fails to delete delegates with bad filepath", func() {
+		err := deleteDelegates("123456789", "bad!file!~?Path$^")
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("delete delegates given good filepath", func() {
+		os.MkdirAll("/opt/cni/bin", 0755)
+		d1 := []byte("blah")
+		os.WriteFile("/opt/cni/bin/123456789", d1, 0644)
+
+		err := deleteDelegates("123456789", "/opt/cni/bin")
+		Expect(err).NotTo(HaveOccurred())
+	})
+})
+
+var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
+	var testNS ns.NetNS
+	var tmpDir string
+	configPath := "/tmp/foo.multus.conf"
+
+	BeforeEach(func() {
+		// Create a new NetNS so we don't modify the host
+		var err error
+		testNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+		os.Setenv("CNI_NETNS", testNS.Path())
+		os.Setenv("CNI_PATH", "/some/path")
+
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		// Touch the default network file.
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+	})
+
+	AfterEach(func() {
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+		Expect(testNS.Close()).To(Succeed())
+		os.Unsetenv("CNI_PATH")
+		os.Unsetenv("CNI_ARGS")
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes delegates", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("executes delegates given faulty namespace", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       "fsdadfad",
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+		// Netns is given garbage value
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).To(MatchError("[//:weave1]: error adding container to network \"weave1\": DelegateAdd: cannot set \"weave-net\" interface name to \"eth0\": validateIfName: no net namespace fsdadfad found: failed to Statfs \"fsdadfad\": no such file or directory"))
+	})
+
+	It("fails to load NetConf with bad json in CmdAdd/Del", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	`),
+		}
+		// Missing close bracket in StdinData
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("executes delegates and cleans up on failure", func() {
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [%s,%s]
+	}`, expectedConf1, expectedConf2)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// This plugin invocation should fail
+		err := fmt.Errorf("expected plugin failure")
+		fExec.addPlugin020(nil, "net1", expectedConf2, nil, err)
+
+		_, err = CmdAdd(args, fExec, nil)
+		Expect(fExec.addIndex).To(Equal(2))
+		Expect(fExec.delIndex).To(Equal(2))
+		Expect(err).To(MatchError("[//:other1]: error adding container to network \"other1\": expected plugin failure"))
+
+	})
+
+	It("executes delegates and cleans up on failure with missing name field", func() {
+		expectedConf1 := `{
+		    "name": "weave1",
+		    "cniVersion": "0.2.0",
+		    "type": "weave-net"
+		}`
+		expectedConf2 := `{
+		    "name": "",
+		    "cniVersion": "0.2.0",
+		    "type": "other-plugin"
+		}`
+		// took out the name in expectedConf2, expecting a new value to be filled in by CmdAdd
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+		    "name": "node-cni-network",
+		    "type": "multus",
+		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "defaultnetworkwaitseconds": 3,
+		    "delegates": [%s,%s]
+		}`, expectedConf1, expectedConf2)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// This plugin invocation should fail
+		err := fmt.Errorf("expected plugin failure")
+		fExec.addPlugin020(nil, "net1", expectedConf2, nil, err)
+
+		_, err = CmdAdd(args, fExec, nil)
+		Expect(fExec.addIndex).To(Equal(1))
+		Expect(fExec.delIndex).To(Equal(2))
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("executes delegates and kubernetes networks with events check", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "net1,net2", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "0.2.0"
+	}`
+		net2 := `{
+		"name": "net2",
+		"type": "mynet2",
+		"cniVersion": "0.2.0"
+	}`
+		net3 := `{
+		"name": "net3",
+		"type": "mynet3",
+		"cniVersion": "0.2.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin020(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+		fExec.addPlugin020(nil, "net2", net2, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.4/24"),
+			},
+		}, nil)
+
+		clientInfo := NewFakeClientInfo()
+		_, err := clientInfo.Client.CoreV1().Pods(fakePod.ObjectMeta.Namespace).Create(
+			context.TODO(), fakePod, metav1.CreateOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net2", net2))
+		Expect(err).NotTo(HaveOccurred())
+		// net3 is not used; make sure it's not accessed
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net3", net3))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, clientInfo)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		recorder := clientInfo.EventRecorder.(*record.FakeRecorder)
+		events := collectEvents(recorder.Events)
+		Expect(len(events)).To(Equal(3))
+		Expect(events[0]).To(Equal("Normal AddedInterface Add eth0 [1.1.1.2/24] from weave1"))
+		Expect(events[1]).To(Equal("Normal AddedInterface Add net1 [1.1.1.3/24] from test/net1"))
+		Expect(events[2]).To(Equal("Normal AddedInterface Add net2 [1.1.1.4/24] from test/net2"))
+	})
+
+	It("executes kubernetes networks and delete it after pod removal", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "0.2.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin020(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		_, err := fKubeClient.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = fKubeClient.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		// set fKubeClient to nil to emulate no pod info
+		err = fKubeClient.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("executes clusterNetwork delegate", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "0.2.0"
+	}`
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin020(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("Verify the cache is created in dataDir", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "0.2.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin020(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err = fKubeClient.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check net count is not incremented")
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("Delete pod without cache", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "0.2.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin020(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err = fKubeClient.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = os.Remove(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check pod/net count is incremented")
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("fails to execute confListDel given no 'plugins' key", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		fakeMultusNetConf := types.NetConf{
+			BinDir: "/opt/cni/bin",
+		}
+		// use fExec for the exec param
+		rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
+		k8sargs, err := k8sclient.GetK8sArgs(args)
+		n, err := types.LoadNetConf(args.StdinData)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
+
+		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
+		Expect(err).To(HaveOccurred())
+	})
+
+})

pkg/multus/multus_cni100_test.go

@@ -0,0 +1,988 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multus
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/testutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
+	var testNS ns.NetNS
+	var tmpDir string
+	resultCNIVersion := "1.0.0"
+	configPath := "/tmp/foo.multus.conf"
+
+	BeforeEach(func() {
+		// Create a new NetNS so we don't modify the host
+		var err error
+		testNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+		os.Setenv("CNI_NETNS", testNS.Path())
+		os.Setenv("CNI_PATH", "/some/path")
+
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		// Touch the default network file.
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+
+	})
+
+	AfterEach(func() {
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+		Expect(testNS.Close()).To(Succeed())
+		os.Unsetenv("CNI_PATH")
+		os.Unsetenv("CNI_ARGS")
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes delegates with CNI Check", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "0.4.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdCheck(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("executes delegates given faulty namespace", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       "fsdadfad",
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+		// Netns is given garbage value
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).To(MatchError("[//:weave1]: error adding container to network \"weave1\": DelegateAdd: cannot set \"weave-net\" interface name to \"eth0\": validateIfName: no net namespace fsdadfad found: failed to Statfs \"fsdadfad\": no such file or directory"))
+	})
+
+	It("returns the previous result using CmdCheck", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+		"plugins": [{
+	        "type": "weave-net",
+	        "cniVersion": "1.0.0",
+		"name": "weave-net-name"
+		}]
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+		"plugins": [{
+	        "type": "other-plugin",
+	        "cniVersion": "1.0.0",
+		"name": "other-name"
+		}]
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdCheck(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("fails to load NetConf with bad json in CmdAdd/Del", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	`),
+		}
+		// Missing close bracket in StdinData
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("executes delegates and cleans up on failure", func() {
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [%s,%s]
+	}`, expectedConf1, expectedConf2)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// This plugin invocation should fail
+		err := fmt.Errorf("expected plugin failure")
+		fExec.addPlugin100(nil, "net1", expectedConf2, nil, err)
+
+		_, err = CmdAdd(args, fExec, nil)
+		Expect(fExec.addIndex).To(Equal(2))
+		Expect(fExec.delIndex).To(Equal(2))
+		Expect(err).To(MatchError("[//:other1]: error adding container to network \"other1\": expected plugin failure"))
+	})
+
+	It("executes delegates and cleans up on failure with missing name field", func() {
+		expectedConf1 := `{
+		    "name": "weave1",
+		    "cniVersion": "1.0.0",
+		    "type": "weave-net"
+		}`
+		expectedConf2 := `{
+		    "name": "",
+		    "cniVersion": "1.0.0",
+		    "type": "other-plugin"
+		}`
+		// took out the name in expectedConf2, expecting a new value to be filled in by CmdAdd
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+		    "name": "node-cni-network",
+		    "type": "multus",
+		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "defaultnetworkwaitseconds": 3,
+		    "delegates": [%s,%s]
+		}`, expectedConf1, expectedConf2)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// This plugin invocation should fail
+		err := fmt.Errorf("missing network name")
+		fExec.addPlugin100(nil, "net1", expectedConf2, nil, err)
+
+		_, err = CmdAdd(args, fExec, nil)
+		Expect(fExec.addIndex).To(Equal(1))
+		Expect(fExec.delIndex).To(Equal(1))
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("executes delegates with runtimeConfigs", func() {
+		podNet := `[{"name":"net1",
+                             "mac": "c2:11:22:33:44:66",
+                             "ips": [ "10.0.0.1" ],
+                             "bandwidth": {
+				     "ingressRate": 2048,
+				     "ingressBurst": 1600,
+				     "egressRate": 4096,
+				     "egressBurst": 1600
+			     },
+			     "portMappings": [
+			     {
+				     "hostPort": 8080, "containerPort": 80, "protocol": "tcp"
+			     },
+			     {
+				     "hostPort": 8000, "containerPort": 8001, "protocol": "udp"
+			     }]
+		     }
+	]`
+		fakePod := testhelpers.NewFakePod("testpod", podNet, "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"capabilities": {"mac": true, "ips": true, "bandwidth": true, "portMappings": true},
+		"cniVersion": "1.0.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: resultCNIVersion,
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		expectedNet1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"capabilities": {
+			"mac": true,
+			"ips": true,
+			"bandwidth": true,
+			"portMappings": true
+		},
+		"runtimeConfig": {
+			"ips": [ "10.0.0.1" ],
+			"mac": "c2:11:22:33:44:66",
+			"bandwidth": {
+				"ingressRate": 2048,
+				"ingressBurst": 1600,
+				"egressRate": 4096,
+				"egressBurst": 1600
+			},
+			"portMappings": [
+			{
+				"hostPort": 8080,
+				"containerPort": 80,
+				"protocol": "tcp"
+			},
+			{
+				"hostPort": 8000,
+				"containerPort": 8001,
+				"protocol": "udp"
+			}]
+		},
+		"cniVersion": "1.0.0"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "net1", expectedNet1, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+			},
+		}, nil)
+
+		clientInfo := NewFakeClientInfo()
+		_, err := clientInfo.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, clientInfo)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*cni100.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+	})
+
+	It("executes delegates and kubernetes networks", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "net1,net2", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		net2 := `{
+		"name": "net2",
+		"type": "mynet2",
+		"cniVersion": "1.0.0"
+	}`
+		net3 := `{
+		"name": "net3",
+		"type": "mynet3",
+		"cniVersion": "1.0.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "net1", net1, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+			},
+		}, nil)
+		fExec.addPlugin100(nil, "net2", net2, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.4/24"),
+			},
+			},
+		}, nil)
+
+		clientInfo := NewFakeClientInfo()
+		_, err := clientInfo.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net2", net2))
+		Expect(err).NotTo(HaveOccurred())
+		// net3 is not used; make sure it's not accessed
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net3", net3))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, clientInfo)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+	})
+
+	It("executes kubernetes networks and delete it after pod removal", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "net1", net1, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+			},
+		}, nil)
+
+		clientInfo := NewFakeClientInfo()
+		_, err := clientInfo.AddPod(fakePod)
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = clientInfo.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, clientInfo)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		// set fKubeClient to nil to emulate no pod info
+		clientInfo.DeletePod(fakePod.ObjectMeta.Namespace, fakePod.ObjectMeta.Name)
+		err = CmdDel(args, fExec, clientInfo)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("ensure delegates get portmap runtime config", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "delegates": [{
+	        "cniVersion": "1.0.0",
+	        "name": "mynet-confList",
+			"plugins": [
+				{
+					"type": "firstPlugin",
+	                "capabilities": {"portMappings": true}
+	            }
+			]
+		}],
+		"runtimeConfig": {
+	        "portMappings": [
+	            {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			]
+	    }
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedConf1 := `{
+	    "capabilities": {"portMappings": true},
+		"name": "mynet-confList",
+	    "cniVersion": "1.0.0",
+	    "type": "firstPlugin",
+	    "runtimeConfig": {
+		    "portMappings": [
+	            {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+			]
+	    }
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, nil, nil)
+		_, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes clusterNetwork delegate", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("Verify the cache is created in dataDir", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "net1", net1, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+			},
+		}, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err = fKubeClient.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check net count is not incremented")
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("Delete pod without cache", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "net1", net1, &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+			},
+		}, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err = fKubeClient.AddNetAttachDef(
+			testhelpers.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = os.Remove(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check pod/net count is incremented")
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("fails to execute confListDel given no 'plugins' key", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
+			},
+			},
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		fakeMultusNetConf := types.NetConf{
+			BinDir: "/opt/cni/bin",
+		}
+		// use fExec for the exec param
+		rawnetconflist := []byte(`{"cniVersion":"1.0.0","name":"weave1","type":"weave-net"}`)
+		k8sargs, err := k8sclient.GetK8sArgs(args)
+		n, err := types.LoadNetConf(args.StdinData)
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
+
+		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
+		Expect(err).To(HaveOccurred())
+	})
+})

pkg/multus/multus_suite_test.go

@@ -0,0 +1,244 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multus
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	cni020 "github.com/containernetworking/cni/pkg/types/020"
+	cni040 "github.com/containernetworking/cni/pkg/types/040"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/record"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMultus(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "multus")
+}
+
+type fakePlugin struct {
+	expectedEnv    []string
+	expectedConf   string
+	expectedIfname string
+	result         cnitypes.Result
+	err            error
+}
+
+type fakeExec struct {
+	cniversion.PluginDecoder
+
+	addIndex        int
+	delIndex        int
+	chkIndex        int
+	expectedDelSkip int
+	plugins         map[string]*fakePlugin
+}
+
+func newFakeExec() *fakeExec {
+	return &fakeExec{
+		plugins: map[string]*fakePlugin{},
+	}
+}
+
+func (f *fakeExec) addPlugin100(expectedEnv []string, expectedIfname, expectedConf string, result *cni100.Result, err error) {
+	f.plugins[expectedIfname] = &fakePlugin{
+		expectedEnv:    expectedEnv,
+		expectedConf:   expectedConf,
+		expectedIfname: expectedIfname,
+		result:         result,
+		err:            err,
+	}
+	if err != nil && err.Error() == "missing network name" {
+		f.expectedDelSkip++
+	}
+}
+
+func (f *fakeExec) addPlugin040(expectedEnv []string, expectedIfname, expectedConf string, result *cni040.Result, err error) {
+	f.plugins[expectedIfname] = &fakePlugin{
+		expectedEnv:    expectedEnv,
+		expectedConf:   expectedConf,
+		expectedIfname: expectedIfname,
+		result:         result,
+		err:            err,
+	}
+	if err != nil && err.Error() == "missing network name" {
+		f.expectedDelSkip++
+	}
+}
+
+func (f *fakeExec) addPlugin020(expectedEnv []string, expectedIfname, expectedConf string, result *cni020.Result, err error) {
+	f.plugins[expectedIfname] = &fakePlugin{
+		expectedEnv:    expectedEnv,
+		expectedConf:   expectedConf,
+		expectedIfname: expectedIfname,
+		result:         result,
+		err:            err,
+	}
+	if err != nil && err.Error() == "missing network name" {
+		f.expectedDelSkip++
+	}
+}
+
+func matchArray(a1, a2 []string) {
+	Expect(len(a1)).To(Equal(len(a2)))
+	for _, e1 := range a1 {
+		found := ""
+		for _, e2 := range a2 {
+			if e1 == e2 {
+				found = e2
+				break
+			}
+		}
+		// Compare element values for more descriptive test failure
+		Expect(e1).To(Equal(found))
+	}
+}
+
+// When faking plugin execution the ExecPlugin() call environ is not populated
+// (while it would be for real exec). Filter the environment variables for
+// CNI-specific ones that testcases will care about.
+func gatherCNIEnv(environ []string) []string {
+	filtered := make([]string, 0)
+	for _, v := range environ {
+		if strings.HasPrefix(v, "CNI_") {
+			filtered = append(filtered, v)
+		}
+	}
+	return filtered
+}
+
+func ParseEnvironment(environ []string) map[string]string {
+	m := map[string]string{}
+
+	for _, e := range environ {
+		if e != "" {
+			parts := strings.SplitN(e, "=", 2)
+			ExpectWithOffset(2, len(parts)).To(Equal(2))
+			m[parts[0]] = parts[1]
+		}
+	}
+	return m
+}
+
+func (f *fakeExec) ExecPlugin(_ context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
+	envMap := ParseEnvironment(environ)
+	cmd := envMap["CNI_COMMAND"]
+	var index int
+	var err error
+	var resultJSON []byte
+
+	switch cmd {
+	case "ADD":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.addIndex))
+		index = f.addIndex
+		f.addIndex++
+	case "CHECK":
+		Expect(len(f.plugins)).To(BeNumerically("==", f.addIndex))
+		index = f.chkIndex
+		f.chkIndex++
+	case "DEL":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.delIndex))
+		index = len(f.plugins) - f.expectedDelSkip - f.delIndex - 1
+		f.delIndex++
+	default:
+		// Should never be reached
+		Expect(false).To(BeTrue())
+	}
+	plugin := f.plugins[envMap["CNI_IFNAME"]]
+
+	//GinkgoT().Logf("[%s %d] exec plugin %q found %+v\n", cmd, index, pluginPath, plugin)
+	fmt.Printf("[%s %d] exec plugin %q found %+v\n", cmd, index, pluginPath, plugin)
+
+	// strip prevResult from stdinData; tests don't need it
+	var m map[string]interface{}
+	reader := strings.NewReader(string(stdinData))
+	writer := new(bytes.Buffer)
+
+	dec := json.NewDecoder(reader)
+	enc := json.NewEncoder(writer)
+	err = dec.Decode(&m)
+	Expect(err).NotTo(HaveOccurred())
+	for k := range m {
+		if k == "prevResult" {
+			delete(m, k)
+		}
+	}
+	err = enc.Encode(&m)
+	Expect(err).NotTo(HaveOccurred())
+
+	if plugin.expectedConf != "" {
+		Expect(writer).To(MatchJSON(plugin.expectedConf))
+	}
+	if plugin.expectedIfname != "" {
+		Expect(envMap["CNI_IFNAME"]).To(Equal(plugin.expectedIfname))
+	}
+
+	if len(plugin.expectedEnv) > 0 {
+		cniEnv := gatherCNIEnv(environ)
+		for _, expectedCniEnvVar := range plugin.expectedEnv {
+			Expect(cniEnv).To(ContainElement(expectedCniEnvVar))
+		}
+	}
+
+	if plugin.err != nil {
+		return nil, plugin.err
+	}
+
+	resultJSON, err = json.Marshal(plugin.result)
+	Expect(err).NotTo(HaveOccurred())
+	return resultJSON, nil
+}
+
+func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) {
+	Expect(len(paths)).To(BeNumerically(">", 0))
+	return filepath.Join(paths[0], plugin), nil
+}
+
+// NewFakeClientInfo returns fake client (just for testing)
+func NewFakeClientInfo() *k8sclient.ClientInfo {
+	return &k8sclient.ClientInfo{
+		Client:        fake.NewSimpleClientset(),
+		NetClient:     netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		EventRecorder: record.NewFakeRecorder(10),
+	}
+}
+
+func collectEvents(source <-chan string) []string {
+	done := false
+	events := make([]string, 0)
+	for !done {
+		select {
+		case ev := <-source:
+			events = append(events, ev)
+		default:
+			done = true
+		}
+	}
+	return events
+}

pkg/netutils/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/netutils/netutils.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2019 Multus Authors
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,27 +12,25 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 package netutils
 
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net"
+	"os"
 	"path/filepath"
 
 	"github.com/containernetworking/cni/libcni"
-	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 )
 
 // DeleteDefaultGW removes the default gateway from marked interfaces.
-func DeleteDefaultGW(args *skel.CmdArgs, ifName string) error {
-	netns, err := ns.GetNS(args.Netns)
+func DeleteDefaultGW(netnsPath string, ifName string) error {
+	netns, err := ns.GetNS(netnsPath)
 	if err != nil {
 		return logging.Errorf("DeleteDefaultGW: Error getting namespace %v", err)
 	}
@@ -52,9 +51,9 @@ func DeleteDefaultGW(args *skel.CmdArgs, ifName string) error {
 }
 
 // SetDefaultGW adds a default gateway on a specific interface
-func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP) error {
+func SetDefaultGW(netnsPath string, ifName string, gateways []net.IP) error {
 	// This ensures we're acting within the net namespace for the pod.
-	netns, err := ns.GetNS(args.Netns)
+	netns, err := ns.GetNS(netnsPath)
 	if err != nil {
 		return logging.Errorf("SetDefaultGW: Error getting namespace %v", err)
 	}
@@ -93,10 +92,10 @@ func SetDefaultGW(args *skel.CmdArgs, ifName string, gateways []net.IP) error {
 }
 
 // DeleteDefaultGWCache updates libcni cache to remove default gateway routes in result
-func DeleteDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, ipv4, ipv6 bool) error {
+func DeleteDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, _ string, ipv4, ipv6 bool) error {
 	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
 
-	cache, err := ioutil.ReadFile(cacheFile)
+	cache, err := os.ReadFile(cacheFile)
 	if err != nil {
 		return err
 	}
@@ -107,7 +106,7 @@ func DeleteDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName strin
 	}
 
 	logging.Debugf("DeleteDefaultGWCache: update cache to delete GW: %s", string(newCache))
-	return ioutil.WriteFile(cacheFile, newCache, 0600)
+	return os.WriteFile(cacheFile, newCache, 0600)
 }
 
 func deleteDefaultGWCacheBytes(cacheFile []byte, ipv4, ipv6 bool) ([]byte, error) {
@@ -265,10 +264,10 @@ func deleteDefaultGWResult020(result map[string]interface{}, ipv4, ipv6 bool) (m
 }
 
 // AddDefaultGWCache updates libcni cache to add default gateway result
-func AddDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, ifName string, gw []net.IP) error {
+func AddDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string, _ string, gw []net.IP) error {
 	cacheFile := filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName))
 
-	cache, err := ioutil.ReadFile(cacheFile)
+	cache, err := os.ReadFile(cacheFile)
 	if err != nil {
 		return err
 	}
@@ -279,7 +278,7 @@ func AddDefaultGWCache(cacheDir string, rt *libcni.RuntimeConf, netName string,
 	}
 
 	logging.Debugf("AddDefaultGWCache: update cache to add GW: %s", string(newCache))
-	return ioutil.WriteFile(cacheFile, newCache, 0600)
+	return os.WriteFile(cacheFile, newCache, 0600)
 }
 
 func addDefaultGWCacheBytes(cacheFile []byte, gw []net.IP) ([]byte, error) {

pkg/netutils/netutils_test.go

@@ -1,4 +1,5 @@
-// Copyright (c) 2019 Multus Authors
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 package netutils
 
@@ -22,13 +22,14 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types/020"
-	"github.com/containernetworking/cni/pkg/types/current"
+	cni040 "github.com/containernetworking/cni/pkg/types/040"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
 	"github.com/vishvananda/netlink"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
@@ -98,11 +99,11 @@ func test020ResultHasIPv6DefaultRoute(data []byte) bool {
 	return false
 }
 
-func testResultHasIPv4DefaultRoute(data []byte) bool {
-	resultRaw, err := current.NewResult(data)
+func test040ResultHasIPv4DefaultRoute(data []byte) bool {
+	resultRaw, err := cni040.NewResult(data)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
-	result, err := current.GetResult(resultRaw)
+	result, err := cni040.GetResult(resultRaw)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 	for _, r := range result.Routes {
@@ -113,11 +114,41 @@ func testResultHasIPv4DefaultRoute(data []byte) bool {
 	return false
 }
 
-func testResultHasIPv6DefaultRoute(data []byte) bool {
-	resultRaw, err := current.NewResult(data)
+func test040ResultHasIPv6DefaultRoute(data []byte) bool {
+	resultRaw, err := cni040.NewResult(data)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
-	result, err := current.GetResult(resultRaw)
+	result, err := cni040.GetResult(resultRaw)
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	for _, r := range result.Routes {
+		if r.Dst.String() == "::/0" {
+			return true
+		}
+	}
+	return false
+}
+
+func test100ResultHasIPv4DefaultRoute(data []byte) bool {
+	resultRaw, err := cni100.NewResult(data)
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	result, err := cni100.GetResult(resultRaw)
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	for _, r := range result.Routes {
+		if r.Dst.String() == "0.0.0.0/0" {
+			return true
+		}
+	}
+	return false
+}
+
+func test100ResultHasIPv6DefaultRoute(data []byte) bool {
+	resultRaw, err := cni100.NewResult(data)
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	result, err := cni100.GetResult(resultRaw)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 	for _, r := range result.Routes {
@@ -198,7 +229,7 @@ var _ = Describe("netutil netlink function testing", func() {
 			Expect(originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				Expect(DeleteDefaultGW(args, IFNAME)).Should(Succeed())
+				Expect(DeleteDefaultGW(args.Netns, IFNAME)).Should(Succeed())
 				return nil
 			})).Should(Succeed())
 		})
@@ -232,7 +263,7 @@ var _ = Describe("netutil netlink function testing", func() {
 			Expect(originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				Expect(SetDefaultGW(args, IFNAME, []net.IP{net.ParseIP("10.0.0.1")})).Should(Succeed())
+				Expect(SetDefaultGW(args.Netns, IFNAME, []net.IP{net.ParseIP("10.0.0.1")})).Should(Succeed())
 				return nil
 			})).Should(Succeed())
 		})
@@ -398,7 +429,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -444,8 +475,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := deleteDefaultGWCacheBytes(origResult, true, false)
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -472,7 +503,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -518,8 +549,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := deleteDefaultGWCacheBytes(origResult, false, true)
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -536,6 +567,150 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(5))
 		})
 
+		It("verify ipv4 default gateway is removed from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "20.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "0.0.0.0/0",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "30.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "20::0:0:0/56",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "::0/0",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "30::0:0:0/64",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult, err := deleteDefaultGWCacheBytes(origResult, true, false)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(5))
+		})
+
+		It("verify ipv6 default gateway is removed from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "20.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "0.0.0.0/0",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "30.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "20::0:0:0/56",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "::0/0",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "30::0:0:0/64",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult, err := deleteDefaultGWCacheBytes(origResult, false, true)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(5))
+		})
+
 		It("verify ipv4 default gateway is added to CNI 0.1.0/0.2.0 results without routes", func() {
 			origResult := []byte(`{
   "kind": "cniCacheV1",
@@ -798,7 +973,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -818,8 +993,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10.1.1.1")})
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -846,7 +1021,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -884,8 +1059,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10.1.1.1")})
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -912,7 +1087,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -932,8 +1107,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10::1:1:1")})
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -960,7 +1135,7 @@ var _ = Describe("netutil cnicache function testing", func() {
       {
         "mac": "0a:c2:e6:3d:45:17",
         "name": "net1",
-        "sandbox": "/var/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
       }
     ],
     "ips": [
@@ -1002,8 +1177,8 @@ var _ = Describe("netutil cnicache function testing", func() {
 			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10::1:1:1")})
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(testResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
-			Expect(testResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test040ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
 
 			// Simplified CNI Cache with 0.3.0/0.3.1/0.4.0 Result
 			type CNICacheResult030_040 struct {
@@ -1020,5 +1195,229 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(6))
 		})
 
+		It("verify ipv4 default gateway is added to CNI 1.0.0 results without routes", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ]
+  }
+}`)
+			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10.1.1.1")})
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(1))
+		})
+
+		It("verify ipv4 default gateway is added to CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "20.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "30.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "20::0:0:0/56",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "30::0:0:0/64",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10.1.1.1")})
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(5))
+		})
+
+		It("verify ipv6 default gateway is added to CNI 1.0.0 results without routes", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ]
+  }
+}`)
+			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10::1:1:1")})
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeFalse())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(1))
+		})
+
+		It("verify ipv6 default gateway is added to CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      },
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "20.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "30.0.0.0/24",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "0.0.0.0/0",
+        "gw": "10.1.1.1"
+      },
+      {
+        "dst": "20::0:0:0/56",
+        "gw": "10::1:1:1"
+      },
+      {
+        "dst": "30::0:0:0/64",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult, err := addDefaultGWCacheBytes(origResult, []net.IP{net.ParseIP("10::1:1:1")})
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(test100ResultHasIPv4DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+			Expect(test100ResultHasIPv6DefaultRoute(testGetResultFromCache(newResult))).To(BeTrue())
+
+			// Simplified CNI Cache with 1.0.0 Result
+			type CNICacheResult100 struct {
+				Kind   string `json:"kind"`
+				Result struct {
+					Routes []struct {
+						Dst string `json:"dst"`
+						Gw  string `json:"gw"`
+					} `json:"routes"`
+				} `json:"result"`
+			}
+			result := CNICacheResult100{}
+			Expect(json.Unmarshal(newResult, &result)).NotTo(HaveOccurred())
+			Expect(len(result.Result.Routes)).To(Equal(6))
+		})
+
 	})
 })

pkg/server/api/api.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+)
+
+const (
+	// MultusCNIAPIEndpoint is an endpoint for multus CNI request (for multus-shim)
+	MultusCNIAPIEndpoint = "/cni"
+	// MultusDelegateAPIEndpoint is an endpoint for multus delegate request (for hotplug)
+	MultusDelegateAPIEndpoint = "/delegate"
+	defaultMultusRunDir       = "/run/multus/"
+
+	// MultusHealthAPIEndpoint is an endpoint API clients can query to know if they can communicate w/ multus server
+	MultusHealthAPIEndpoint = "/healthz"
+)
+
+// DoCNI sends a CNI request to the CNI server via JSON + HTTP over a root-owned unix socket,
+// and returns the result
+func DoCNI(url string, req interface{}, socketPath string) ([]byte, error) {
+	data, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal CNI request %v: %v", req, err)
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Dial: func(proto, addr string) (net.Conn, error) {
+				return net.Dial("unix", socketPath)
+			},
+		},
+	}
+
+	resp, err := client.Post(url, "application/json", bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send CNI request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CNI result: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("CNI request failed with status %v: '%s'", resp.StatusCode, string(body))
+	}
+
+	return body, nil
+}
+
+// GetAPIEndpoint returns endpoint URL for multus-daemon
+func GetAPIEndpoint(endpoint string) string {
+	return fmt.Sprintf("http://dummy%s", endpoint)
+}
+
+// CreateDelegateRequest creates Request for delegate API request
+func CreateDelegateRequest(cniCommand, cniContainerID, cniNetNS, cniIFName, podNamespace, podName, podUID string, cniConfig []byte, interfaceAttributes *DelegateInterfaceAttributes) *Request {
+	return &Request{
+		Env: map[string]string{
+			"CNI_COMMAND":     strings.ToUpper(cniCommand),
+			"CNI_CONTAINERID": cniContainerID,
+			"CNI_NETNS":       cniNetNS,
+			"CNI_IFNAME":      cniIFName,
+			"CNI_ARGS":        fmt.Sprintf("K8S_POD_NAMESPACE=%s;K8S_POD_NAME=%s;K8S_POD_UID=%s", podNamespace, podName, podUID),
+		},
+		Config:              cniConfig,
+		InterfaceAttributes: interfaceAttributes,
+	}
+}

pkg/server/api/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package api is the package that contains thick pluigin's server apis.
+package api

pkg/server/api/shim.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+)
+
+// ShimNetConf for the SHIM cni config file written in json
+type ShimNetConf struct {
+	// Note: This struct contains NetConf in pkg/types, but this struct is only used to parse
+	// following fields, so we skip to include NetConf here. Other fields are directly send to
+	// multus-daemon as a part of skel.CmdArgs, StdinData.
+	// types.NetConf
+
+	CNIVersion      string `json:"cniVersion,omitempty"`
+	MultusSocketDir string `json:"daemonSocketDir"`
+	LogFile         string `json:"logFile,omitempty"`
+	LogLevel        string `json:"logLevel,omitempty"`
+	LogToStderr     bool   `json:"logToStderr,omitempty"`
+}
+
+// CmdAdd implements the CNI spec ADD command handler
+func CmdAdd(args *skel.CmdArgs) error {
+	response, cniVersion, err := postRequest(args)
+	if err != nil {
+		return logging.Errorf("CmdAdd (shim): %v", err)
+	}
+
+	logging.Verbosef("CmdAdd (shim): %v", *response.Result)
+	return cnitypes.PrintResult(response.Result, cniVersion)
+}
+
+// CmdCheck implements the CNI spec CHECK command handler
+func CmdCheck(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args)
+	if err != nil {
+		return logging.Errorf("CmdCheck (shim): %v", err)
+	}
+
+	return err
+}
+
+// CmdDel implements the CNI spec DEL command handler
+func CmdDel(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args)
+	if err != nil {
+		return logging.Errorf("CmdDel (shim): %v", err)
+	}
+	return nil
+}
+
+func postRequest(args *skel.CmdArgs) (*Response, string, error) {
+	multusShimConfig, err := shimConfig(args.StdinData)
+	if err != nil {
+		return nil, "", fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err)
+	}
+
+	cniRequest, err := newCNIRequest(args)
+	if err != nil {
+		return nil, multusShimConfig.CNIVersion, err
+	}
+
+	body, err := DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
+	if err != nil {
+		return nil, multusShimConfig.CNIVersion, err
+	}
+
+	response := &Response{}
+	if len(body) != 0 {
+		if err = json.Unmarshal(body, response); err != nil {
+			err = fmt.Errorf("failed to unmarshal response '%s': %v", string(body), err)
+			return nil, multusShimConfig.CNIVersion, err
+		}
+	}
+	return response, multusShimConfig.CNIVersion, nil
+}
+
+// Create and fill a Request with this Plugin's environment and stdin which
+// contain the CNI variables and configuration
+func newCNIRequest(args *skel.CmdArgs) (*Request, error) {
+	envMap := make(map[string]string)
+	for _, item := range os.Environ() {
+		idx := strings.Index(item, "=")
+		if idx > 0 {
+			envMap[strings.TrimSpace(item[:idx])] = item[idx+1:]
+		}
+	}
+
+	return &Request{
+		Env:    envMap,
+		Config: args.StdinData,
+	}, nil
+}
+
+func shimConfig(cniConfig []byte) (*ShimNetConf, error) {
+	multusConfig := &ShimNetConf{}
+	if err := json.Unmarshal(cniConfig, multusConfig); err != nil {
+		return nil, fmt.Errorf("failed to gather the multus configuration: %w", err)
+	}
+	if multusConfig.MultusSocketDir == "" {
+		multusConfig.MultusSocketDir = defaultMultusRunDir
+	}
+	// Logging
+	logging.SetLogStderr(multusConfig.LogToStderr)
+	if multusConfig.LogFile != "" {
+		logging.SetLogFile(multusConfig.LogFile)
+	}
+	if multusConfig.LogLevel != "" {
+		logging.SetLogLevel(multusConfig.LogLevel)
+	}
+	return multusConfig, nil
+}

pkg/server/api/socket.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"path/filepath"
+)
+
+const (
+	serverSocketName = "multus.sock"
+)
+
+// SocketPath returns the path of the multus CNI socket
+func SocketPath(rundir string) string {
+	return filepath.Join(rundir, serverSocketName)
+}

pkg/server/api/types.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+)
+
+// Request sent to the Server by the multus-shim
+type Request struct {
+	// CNI environment variables, like CNI_COMMAND and CNI_NETNS
+	Env map[string]string `json:"env,omitempty"`
+	// CNI configuration passed via stdin to the CNI plugin
+	Config []byte `json:"config,omitempty"`
+	// Annotation for Delegate request
+	InterfaceAttributes *DelegateInterfaceAttributes `json:"interfaceAttributes,omitempty"`
+}
+
+// DelegateInterfaceAttributes annotates delegate request for additional config
+type DelegateInterfaceAttributes struct {
+	// IPRequest contains an optional requested IP address for this network
+	// attachment
+	IPRequest []string `json:"ips,omitempty"`
+	// MacRequest contains an optional requested MAC address for this
+	// network attachment
+	MacRequest string `json:"mac,omitempty"`
+	// CNIArgs contains additional CNI arguments for the network interface
+	CNIArgs *map[string]interface{} `json:"cni-args"`
+}
+
+// Response represents the response (computed in the CNI server) for
+// ADD / DEL / CHECK for a Pod.
+type Response struct {
+	Result *cni100.Result
+}

pkg/server/config/config_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestConfig(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "server/config")
+}

pkg/server/config/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2022 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package config is the package that contains multus cni config related
-// utilities.
+// Package config is the package that contains the libraries that operates
+// CNI config files
 package config

pkg/server/config/generator.go

@@ -0,0 +1,213 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+)
+
+const (
+	configListCapabilityKey   = "plugins"
+	multusPluginName          = "multus-shim"
+	singleConfigCapabilityKey = "capabilities"
+)
+
+// Option mutates the `conf` object
+type Option func(conf *MultusConf) error
+
+// MultusConf holds the multus configuration
+type MultusConf struct {
+	BinDir                   string              `json:"binDir,omitempty"`
+	Capabilities             map[string]bool     `json:"capabilities,omitempty"`
+	CNIVersion               string              `json:"cniVersion"`
+	LogFile                  string              `json:"logFile,omitempty"`
+	LogLevel                 string              `json:"logLevel,omitempty"`
+	LogToStderr              bool                `json:"logToStderr,omitempty"`
+	LogOptions               *logging.LogOptions `json:"logOptions,omitempty"`
+	Name                     string              `json:"name"`
+	ClusterNetwork           string              `json:"clusterNetwork,omitempty"`
+	NamespaceIsolation       bool                `json:"namespaceIsolation,omitempty"`
+	RawNonIsolatedNamespaces string              `json:"globalNamespaces,omitempty"`
+	ReadinessIndicatorFile   string              `json:"readinessindicatorfile,omitempty"`
+	Type                     string              `json:"type"`
+	CniDir                   string              `json:"cniDir,omitempty"`
+	CniConfigDir             string              `json:"cniConfigDir,omitempty"`
+	SocketDir                string              `json:"socketDir,omitempty"`
+	MultusConfigFile         string              `json:"multusConfigFile,omitempty"`
+	MultusMasterCni          string              `json:"multusMasterCNI,omitempty"`
+	MultusAutoconfigDir      string              `json:"multusAutoconfigDir,omitempty"`
+	ForceCNIVersion          bool                `json:"forceCNIVersion,omitempty"`
+	OverrideNetworkName      bool                `json:"overrideNetworkName,omitempty"`
+}
+
+// ParseMultusConfig parses multus config from configPath and create MultusConf.
+func ParseMultusConfig(configPath string) (*MultusConf, error) {
+	config, err := os.ReadFile(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("ParseMultusConfig failed to read the config file's contents: %w", err)
+	}
+
+	multusconf := MultusConf{
+		MultusConfigFile: "auto",
+		Type:             multusPluginName,
+		Capabilities:     map[string]bool{},
+		CniConfigDir:     "/etc/cni/net.d",
+	}
+
+	if err := json.Unmarshal(config, &multusconf); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err)
+	}
+	multusconf.Name = MultusDefaultNetworkName // change name
+
+	return &multusconf, nil
+}
+
+// CheckVersionCompatibility checks compatibilty of the
+// top level cni version with the delegate cni version.
+// Since version 0.4.0, CHECK was introduced, which
+// causes incompatibility.
+func CheckVersionCompatibility(mc *MultusConf, delegate interface{}) error {
+	const versionFmt = "delegate cni version is %s while top level cni version is %s"
+	v040, _ := semver.Make("0.4.0")
+	multusCNIVersion, err := semver.Make(mc.CNIVersion)
+
+	if err != nil {
+		return errors.New("couldn't get top level cni version")
+	}
+
+	if multusCNIVersion.GTE(v040) {
+		delegatesMap, ok := delegate.(map[string]interface{})
+		if !ok {
+			return errors.New("couldn't get cni version of delegate")
+		}
+		delegateVersion, ok := delegatesMap["cniVersion"].(string)
+		if !ok {
+			return errors.New("couldn't get cni version of delegate")
+		}
+		v, err := semver.Make(delegateVersion)
+		if err != nil {
+			return err
+		}
+		if v.LT(v040) {
+			return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
+		}
+	}
+
+	return nil
+}
+
+// Generate generates the multus configuration from whatever state is currently
+// held
+func (mc *MultusConf) Generate() (string, error) {
+	// before marshal, flush variables which is not required for multus-shim config
+	mc.CniConfigDir = ""
+	mc.MultusConfigFile = ""
+	mc.MultusAutoconfigDir = ""
+
+	data, err := json.Marshal(mc)
+	return string(data), err
+}
+
+func (mc *MultusConf) setCapabilities(cniData interface{}) error {
+	var enabledCapabilities []string
+	var pluginsList []interface{}
+	cniDataMap, ok := cniData.(map[string]interface{})
+	if ok {
+		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
+			pluginsList = pluginsListEntry.([]interface{})
+		}
+	} else {
+		return errors.New("couldn't get cni config from delegate")
+	}
+
+	if len(pluginsList) > 0 {
+		for _, pluginData := range pluginsList {
+			enabledCapabilities = append(
+				enabledCapabilities,
+				extractCapabilities(pluginData)...)
+		}
+	} else {
+		enabledCapabilities = extractCapabilities(cniData)
+	}
+
+	for _, capability := range enabledCapabilities {
+		mc.Capabilities[capability] = true
+	}
+	return nil
+}
+
+func extractCapabilities(capabilitiesInterface interface{}) []string {
+	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
+	if !ok {
+		return nil
+	}
+	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	var enabledCapabilities []string
+	if len(capabilities) > 0 {
+		for capName, isCapabilityEnabled := range capabilities {
+			if isCapabilityEnabled.(bool) {
+				enabledCapabilities = append(enabledCapabilities, capName)
+			}
+		}
+	}
+	return enabledCapabilities
+}
+
+func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
+	if remainingTries == 0 {
+		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
+	}
+	var cniPluginConfigs []string
+	files, err := os.ReadDir(cniConfigDirPath)
+	if err != nil {
+		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
+	}
+
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "00-multus") {
+			continue
+		}
+		fileExtension := filepath.Ext(file.Name())
+		if fileExtension == ".conf" || fileExtension == ".conflist" {
+			cniPluginConfigs = append(cniPluginConfigs, file.Name())
+		}
+	}
+
+	if len(cniPluginConfigs) == 0 {
+		time.Sleep(time.Second)
+		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
+	}
+	sort.Strings(cniPluginConfigs)
+	return cniPluginConfigs[0], nil
+}

pkg/server/config/generator_test.go

@@ -0,0 +1,268 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	primaryCNIName = "myCNI"
+	cniVersion     = "0.4.0"
+)
+
+type testCase struct {
+	t                        *testing.T
+	configGenerationFunction func() (string, error)
+}
+
+var primaryCNIConfig = map[string]interface{}{
+	"cniVersion":         "1.0.0",
+	"name":               "ovn-kubernetes",
+	"type":               "ovn-k8s-cni-overlay",
+	"ipam":               "{}",
+	"dns":                "{}",
+	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+	"logLevel":           "5",
+	"logfile-maxsize":    100,
+	"logfile-maxbackups": 5,
+	"logfile-maxage":     5,
+}
+var primaryCNIFile = "/etc/cni/net.d/10-flannel.conf"
+
+var _ = Describe("Configuration Generator", func() {
+	var tmpDir string
+	var err error
+
+	BeforeEach(func() {
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("basic multus config", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		expectedResult := fmt.Sprintf(`
+			{
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with capabilities", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(documentHelper(`{"capabilities": {"portMappings": true}}`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{
+					"portMappings":true
+				},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with multiple capabilities", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{"portMappings":true,"tuning":true},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with multiple capabilities filter only enabled", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{"portMappings":true},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with multiple capabilities defined on a plugin", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(
+			documentHelper(
+				`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{"portMappings":true,"tuning":true},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with multiple capabilities defined on multiple plugins", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(
+			documentHelper(`
+				{
+					"plugins": [
+					{
+						"capabilities": { "portMappings": true }
+					}, 
+					{
+						"capabilities": { "tuning": true }
+					}
+					]
+				}`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{"portMappings":true,"tuning":true},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+
+	It("multus config with multiple capabilities defined on multiple plugins filter only enabled", func() {
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q,
+			"clusterNetwork": %q
+		}`, primaryCNIName, cniVersion, primaryCNIFile)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", tmpDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConfig, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(multusConfig.setCapabilities(
+			documentHelper(`
+				{
+					"plugins": [
+					{
+						"capabilities": {
+							"portMappings": true
+						}
+					},
+					{
+						"capabilities": {
+							"tuning": false
+						}
+					}
+					]
+				}`))).To(Succeed())
+		expectedResult := fmt.Sprintf(`
+			{
+				"capabilities":{"portMappings":true},
+				"cniVersion":"0.4.0",
+				"clusterNetwork":"%s",
+				"name":"multus-cni-network",
+				"type":"multus-shim"
+			}`, primaryCNIFile)
+		Expect(multusConfig.Generate()).Should(MatchJSON(expectedResult))
+	})
+})
+
+func documentHelper(pluginInfo string) interface{} {
+	dp, _ := documentCNIData([]byte(pluginInfo))
+	return dp
+}
+
+func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
+	}
+	return cniData, nil
+}

pkg/server/config/manager.go

@@ -11,25 +11,24 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 package config
 
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/fsnotify/fsnotify"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 )
 
 // MultusDefaultNetworkName holds the default name of the multus network
 const (
 	multusConfigFileName     = "00-multus.conf"
 	MultusDefaultNetworkName = "multus-cni-network"
-	userRWPermission         = 0600
+	UserRWPermission         = 0600
 )
 
 // Manager monitors the configuration of the primary CNI plugin, and
@@ -43,32 +42,68 @@ type Manager struct {
 	primaryCNIConfigPath string
 }
 
-// NewManager returns a config manager object, configured to persist the
-// configuration to `multusAutoconfigDir`. This constructor will auto-discover
+// NewManager returns a config manager object, configured to read the
+// primary CNI configuration in `multusAutoconfigDir`. This constructor will auto-discover
 // the primary CNI for which it will delegate.
-func NewManager(config MultusConf, multusAutoconfigDir string) (*Manager, error) {
-	defaultCNIPluginName, err := primaryCNIPluginName(multusAutoconfigDir)
+func NewManager(config MultusConf, multusAutoconfigDir string, forceCNIVersion bool) (*Manager, error) {
+	defaultCNIPluginName, err := getPrimaryCNIPluginName(multusAutoconfigDir)
 	if err != nil {
 		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
 		return nil, err
 	}
-	return newManager(config, multusAutoconfigDir, defaultCNIPluginName)
+	return newManager(config, multusAutoconfigDir, defaultCNIPluginName, forceCNIVersion)
 }
 
 // NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
 // configured to persist the configuration to `multusAutoconfigDir`. This
 // constructor will use the primary CNI plugin indicated by the user, via the
 // primaryCNIPluginName variable.
-func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir string, primaryCNIPluginName string) (*Manager, error) {
-	return newManager(config, multusAutoconfigDir, primaryCNIPluginName)
+func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir, primaryCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
+	return newManager(config, multusAutoconfigDir, primaryCNIPluginName, forceCNIVersion)
 }
 
-func newManager(config MultusConf, multusConfigDir string, defaultCNIPluginName string) (*Manager, error) {
+// overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case
+func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
+	masterCNIConfigData, err := os.ReadFile(cniConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read cni config %s: %v", cniConfigFile, err)
+	}
+
+	var primaryCNIConfigData map[string]interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &primaryCNIConfigData); err != nil {
+		return fmt.Errorf("failed to unmarshall cni config %s: %w", cniConfigFile, err)
+	}
+
+	primaryCNIConfigData["cniVersion"] = multusCNIVersion
+	configBytes, err := json.Marshal(primaryCNIConfigData)
+	if err != nil {
+		return fmt.Errorf("couldn't update cluster network config: %v", err)
+	}
+
+	err = os.WriteFile(cniConfigFile, configBytes, 0644)
+	if err != nil {
+		return fmt.Errorf("couldn't update cluster network config: %v", err)
+	}
+	return nil
+}
+
+func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
+	if forceCNIVersion {
+		err := overrideCNIVersion(cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), config.CNIVersion)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	watcher, err := newWatcher(multusConfigDir)
 	if err != nil {
 		return nil, err
 	}
 
+	if defaultCNIPluginName == fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) {
+		return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", multusConfigDir, multusConfigFileName)
+	}
+
 	configManager := &Manager{
 		configWatcher:        watcher,
 		multusConfig:         &config,
@@ -89,6 +124,11 @@ func (m *Manager) loadPrimaryCNIConfigFromFile() error {
 	if err != nil {
 		return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
 	}
+
+	if err = CheckVersionCompatibility(m.multusConfig, primaryCNIConfigData); err != nil {
+		return err
+	}
+
 	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
 }
 
@@ -104,16 +144,16 @@ func (m *Manager) OverrideNetworkName() error {
 	if networkName == "" {
 		return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
 	}
-	return m.multusConfig.Mutate(WithOverriddenName(networkName))
+	m.multusConfig.Name = networkName
+	return nil
 }
 
 func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
 	cniConfigData := primaryCNIConfigData.(map[string]interface{})
 
 	m.cniConfigData = cniConfigData
-	return m.multusConfig.Mutate(
-		withDelegates(cniConfigData),
-		withCapabilities(cniConfigData))
+	m.multusConfig.ClusterNetwork = m.primaryCNIConfigPath
+	return m.multusConfig.setCapabilities(cniConfigData)
 }
 
 // GenerateConfig generates a multus configuration from its current state
@@ -125,10 +165,10 @@ func (m Manager) GenerateConfig() (string, error) {
 	return m.multusConfig.Generate()
 }
 
-// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
+// MonitorPluginConfiguration monitors the configuration file pointed
 // to by the primaryCNIPluginName attribute, and re-generates the multus
 // configuration whenever the primary CNI config is updated.
-func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
+func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<- struct{}) error {
 	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
 
 	for {
@@ -140,6 +180,7 @@ func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, don
 				logging.Debugf("skipping un-related event %v", event)
 				continue
 			}
+			logging.Debugf("process event: %v", event)
 
 			if !shouldRegenerateConfig(event) {
 				continue
@@ -167,7 +208,7 @@ func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, don
 		case <-shutDown:
 			logging.Verbosef("Stopped monitoring, closing channel ...")
 			_ = m.configWatcher.Close()
-			done <- struct{}{}
+			close(done)
 			return nil
 		}
 	}
@@ -176,10 +217,11 @@ func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, don
 // PersistMultusConfig persists the provided configuration to the disc, with
 // Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
 func (m Manager) PersistMultusConfig(config string) error {
-	return ioutil.WriteFile(m.multusConfigFilePath, []byte(config), userRWPermission)
+	logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
+	return os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
 }
 
-func primaryCNIPluginName(multusAutoconfigDir string) (string, error) {
+func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
 	masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
 	if err != nil {
 		return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
@@ -211,12 +253,11 @@ func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
 }
 
 func shouldRegenerateConfig(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Write == fsnotify.Write ||
-		event.Op&fsnotify.Create == fsnotify.Create
+	return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
 }
 
 func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
-	masterCNIConfigData, err := ioutil.ReadFile(masterCNIPluginPath)
+	masterCNIConfigData, err := os.ReadFile(masterCNIPluginPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
 	}

pkg/server/config/manager_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Configuration Manager", func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var configManager *Manager
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	BeforeEach(func() {
+		var err error
+		multusConfigDir, err = os.MkdirTemp("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(os.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed())
+
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q
+		}`, defaultCniConfig, cniVersion)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConf, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+
+		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
+		expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"multus-cni-network\",\"clusterNetwork\":\"%s\",\"type\":\"multus-shim\"}", defaultCniConfig)
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(config).To(Equal(expectedResult))
+	})
+
+	It("Check overrideCNIVersion is worked", func() {
+		err := overrideCNIVersion(defaultCniConfig, "1.1.1")
+		Expect(err).NotTo(HaveOccurred())
+		raw, err := os.ReadFile(defaultCniConfig)
+		Expect(err).NotTo(HaveOccurred())
+
+		var jsonConfig map[string]interface{}
+		err = json.Unmarshal(raw, &jsonConfig)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(jsonConfig["cniVersion"].(string)).To(Equal("1.1.1"))
+	})
+
+	It("Check primaryCNIPlugin can be identified", func() {
+		fileName, err := getPrimaryCNIPluginName(multusConfigDir)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fileName).To(Equal(primaryCNIPluginName))
+	})
+
+	It("Check MonitorPluginConfiguration", func() {
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+
+		err = configManager.PersistMultusConfig(config)
+		Expect(err).NotTo(HaveOccurred())
+
+		configWatcherDoneChannel := make(chan struct{})
+		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
+			err := configManager.MonitorPluginConfiguration(configWatcherDoneChannel, stopChannel)
+			Expect(err).NotTo(HaveOccurred())
+		}(make(chan struct{}), configWatcherDoneChannel)
+
+		updatedCNIConfig := `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni2",
+  "ipam": {},
+  "dns": {}
+}
+`
+		// update the CNI config to update the master config
+		Expect(os.WriteFile(defaultCniConfig, []byte(updatedCNIConfig), UserRWPermission)).To(Succeed())
+
+		// wait for a while to get fsnotify event
+		time.Sleep(100 * time.Millisecond)
+		file, err := os.ReadFile(configManager.multusConfigFilePath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(string(file)).To(Equal(config))
+
+		// stop groutine
+		configWatcherDoneChannel <- struct{}{}
+	})
+
+	When("the user requests the name of the multus configuration to be overridden", func() {
+		BeforeEach(func() {
+			Expect(configManager.OverrideNetworkName()).To(Succeed())
+		})
+
+		It("Overrides the name of the multus configuration when requested", func() {
+			expectedResult := fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"name\":\"mycni-name\",\"clusterNetwork\":\"%s\",\"type\":\"multus-shim\"}", defaultCniConfig)
+			config, err := configManager.GenerateConfig()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(expectedResult))
+		})
+	})
+})
+
+var _ = Describe("Configuration Manager with mismatched cniVersion", func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.3.1",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	It("test cni version incompatibility", func() {
+		var err error
+		multusConfigDir, err = os.MkdirTemp("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(os.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), UserRWPermission)).To(Succeed())
+
+		multusConfFile := fmt.Sprintf(`{
+			"name": %q,
+			"cniVersion": %q
+		}`, defaultCniConfig, cniVersion)
+		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
+		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
+
+		multusConf, err := ParseMultusConfig(multusConfFileName)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		Expect(err).To(MatchError("failed to load the primary CNI configuration as a multus delegate with error 'delegate cni version is 0.3.1 while top level cni version is 0.4.0'"))
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+})

pkg/server/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package server is the package that contains thick pluigin's server libraries.
+package server

pkg/server/exec_chroot.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containernetworking/cni/pkg/invoke"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/version"
+)
+
+// ChrootExec implements invoke.Exec to execute CNI with chroot
+type ChrootExec struct {
+	Stderr     io.Writer
+	chrootDir  string
+	workingDir string   // working directory in the outer root
+	outerRoot  *os.File // outer root directory
+	version.PluginDecoder
+	mu sync.Mutex
+}
+
+var _ invoke.Exec = &ChrootExec{}
+
+func (e *ChrootExec) chroot() error {
+	var err error
+	e.workingDir, err = os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
+		return fmt.Errorf("getwd before chroot failed: %v", err)
+	}
+
+	e.outerRoot, err = os.Open("/")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
+		return fmt.Errorf("getwd before chroot failed: %v", err)
+	}
+
+	if err := syscall.Chroot(e.chrootDir); err != nil {
+		fmt.Fprintf(os.Stderr, "chroot to %s failed: %v\n", e.chrootDir, err)
+		return fmt.Errorf("chroot to %s failed: %v", e.chrootDir, err)
+	}
+
+	if err := os.Chdir("/"); err != nil {
+		fmt.Fprintf(os.Stderr, "chdir to \"/\" failed: %v\n", err)
+		return fmt.Errorf("chdir to \"/\" failed: %v", err)
+	}
+
+	return nil
+}
+
+func (e *ChrootExec) escape() error {
+	if e.outerRoot == nil || e.workingDir == "" {
+		return nil
+	}
+
+	// change directory to outer root and close it
+	if err := syscall.Fchdir(int(e.outerRoot.Fd())); err != nil {
+		fmt.Fprintf(os.Stderr, "changing directory to outer root failed: %v\n", err)
+		return fmt.Errorf("changing directory to outer root failed: %v", err)
+	}
+
+	if err := e.outerRoot.Close(); err != nil {
+		fmt.Fprintf(os.Stderr, "closing outer root failed: %v\n", err)
+		return fmt.Errorf("closing outer root failed: %v", err)
+	}
+
+	// chroot to current directory aka "." being the outer root
+	if err := syscall.Chroot("."); err != nil {
+		fmt.Fprintf(os.Stderr, "chroot to current directory failed: %v\n", err)
+		return fmt.Errorf("chroot to current directory failed: %v", err)
+	}
+
+	if err := os.Chdir(e.workingDir); err != nil {
+		fmt.Fprintf(os.Stderr, "chdir to working directory failed: %v\n", err)
+		return fmt.Errorf("chdir to working directory failed: %v", err)
+	}
+	e.outerRoot = nil
+	e.workingDir = ""
+
+	return nil
+}
+
+// ExecPlugin executes CNI plugin with given environment/stdin data.
+func (e *ChrootExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
+	// lock and do chroot to execute plugin with host root
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	err := e.chroot()
+	defer e.escape()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ExecPlugin failed at chroot: %v\n", err)
+		return nil, fmt.Errorf("ExecPlugin failed at chroot: %v", err)
+	}
+
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	c := exec.CommandContext(ctx, pluginPath)
+	c.Env = environ
+	c.Stdin = bytes.NewBuffer(stdinData)
+	c.Stdout = stdout
+	c.Stderr = stderr
+
+	// Retry the command on "text file busy" errors
+	for i := 0; i <= 5; i++ {
+		err = c.Run()
+
+		// Command succeeded
+		if err == nil {
+			break
+		}
+
+		// If the plugin is currently about to be written, then we wait a
+		// second and try it again
+		if strings.Contains(err.Error(), "text file busy") {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// All other errors except than the busy text file
+		return nil, e.pluginErr(err, stdout.Bytes(), stderr.Bytes())
+	}
+
+	// Copy stderr to caller's buffer in case plugin printed to both
+	// stdout and stderr for some reason. Ignore failures as stderr is
+	// only informational.
+	if e.Stderr != nil && stderr.Len() > 0 {
+		_, _ = stderr.WriteTo(e.Stderr)
+	}
+	return stdout.Bytes(), nil
+}
+
+func (e *ChrootExec) pluginErr(err error, stdout, stderr []byte) error {
+	emsg := types.Error{}
+	if len(stdout) == 0 {
+		if len(stderr) == 0 {
+			emsg.Msg = fmt.Sprintf("netplugin failed with no error message: %v", err)
+		} else {
+			emsg.Msg = fmt.Sprintf("netplugin failed: %q", string(stderr))
+		}
+	} else if perr := json.Unmarshal(stdout, &emsg); perr != nil {
+		emsg.Msg = fmt.Sprintf("netplugin failed but error parsing its diagnostic message %q: %v", string(stdout), perr)
+	}
+	return &emsg
+}
+
+// FindInPath try to find CNI plugin based on given path
+func (e *ChrootExec) FindInPath(plugin string, paths []string) (string, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	err := e.chroot()
+	defer e.escape()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "FindInPath failed at chroot: %v\n", err)
+		return "", fmt.Errorf("FindInPath failed at chroot: %v", err)
+	}
+
+	return invoke.FindInPath(plugin, paths)
+}

pkg/server/exec_chroot_test.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("exec_chroot", func() {
+
+	It("Call ChrootExec.ExecPlugin with dummy", func() {
+		chrootExec := &ChrootExec{
+			Stderr:    os.Stderr,
+			chrootDir: "/usr",
+		}
+
+		_, err := chrootExec.ExecPlugin(context.Background(), "/bin/true", nil, nil)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	It("Call invalid ChrootExec.ExecPlugin with dummy", func() {
+		chrootExec := &ChrootExec{
+			Stderr:    os.Stderr,
+			chrootDir: "/tmp",
+		}
+
+		_, err := chrootExec.ExecPlugin(context.Background(), "/bin/true", nil, nil)
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("Call ChrootExec.FindInPath with dummy", func() {
+		chrootExec := &ChrootExec{
+			Stderr:    os.Stderr,
+			chrootDir: "/usr/bin",
+		}
+
+		_, err := chrootExec.FindInPath("true", []string{"/"})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+})

pkg/server/server.go

@@ -0,0 +1,555 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/containernetworking/cni/pkg/invoke"
+	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	k8s "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+)
+
+const (
+	fullReadWriteExecutePermissions    = 0777
+	thickPluginSocketRunDirPermissions = 0700
+)
+
+// FilesystemPreRequirements ensures the target `rundir` features the correct
+// permissions.
+func FilesystemPreRequirements(rundir string) error {
+	if err := os.RemoveAll(rundir); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove old pod info socket directory %s: %v", rundir, err)
+	}
+	if err := os.MkdirAll(rundir, thickPluginSocketRunDirPermissions); err != nil {
+		return fmt.Errorf("failed to create pod info socket directory %s: %v", rundir, err)
+	}
+	return nil
+}
+
+// HandleCNIRequest is the CNI server handler function; it is invoked whenever
+// a CNI request is processed.
+func (s *Server) HandleCNIRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+	var result []byte
+	var err error
+
+	logging.Verbosef("%s starting CNI request %+v", cmd, cniCmdArgs)
+	switch cmd {
+	case "ADD":
+		result, err = cmdAdd(cniCmdArgs, k8sArgs, exec, kubeClient)
+	case "DEL":
+		err = cmdDel(cniCmdArgs, k8sArgs, exec, kubeClient)
+	case "CHECK":
+		err = cmdCheck(cniCmdArgs, k8sArgs, exec, kubeClient)
+	default:
+		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
+	}
+	logging.Verbosef("%s finished CNI request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
+	if err != nil {
+		// Prefix errors with request info for easier failure debugging
+		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
+	}
+	return result, nil
+}
+
+// HandleDelegateRequest is the CNI server handler function; it is invoked whenever
+// a CNI request is processed as delegate CNI request.
+func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+	var result []byte
+	var err error
+	var multusConfByte []byte
+
+	multusConfByte = bytes.Replace(s.serverConfig, []byte(","), []byte("{"), 1)
+	multusConfig := types.GetDefaultNetConf()
+	if err = json.Unmarshal(multusConfByte, multusConfig); err != nil {
+		return nil, err
+	}
+
+	logging.Verbosef("%s starting delegate request %+v", cmd, cniCmdArgs)
+	switch cmd {
+	case "ADD":
+		result, err = cmdDelegateAdd(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig, interfaceAttributes)
+	case "DEL":
+		err = cmdDelegateDel(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+	case "CHECK":
+		err = cmdDelegateCheck(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+	default:
+		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
+	}
+	logging.Verbosef("%s finished Delegate request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
+	if err != nil {
+		// Prefix errors with request info for easier failure debugging
+		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
+	}
+	return result, nil
+}
+
+// GetListener creates a listener to a unix socket located in `socketPath`
+func GetListener(socketPath string) (net.Listener, error) {
+	l, err := net.Listen("unix", socketPath)
+	if err != nil {
+		return nil, logging.Errorf("failed to listen on pod info socket: %v", err)
+	}
+	if err := os.Chmod(socketPath, config.UserRWPermission); err != nil {
+		_ = l.Close()
+		return nil, logging.Errorf("failed to listen on pod info socket: %v", err)
+	}
+	return l, nil
+}
+
+// NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
+func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte) (*Server, error) {
+	kubeClient, err := k8s.InClusterK8sClient()
+	if err != nil {
+		return nil, fmt.Errorf("error getting k8s client: %v", err)
+	}
+
+	exec := invoke.Exec(nil)
+	if daemonConfig.ChrootDir != "" {
+		chrootExec := &ChrootExec{
+			Stderr:    os.Stderr,
+			chrootDir: daemonConfig.ChrootDir,
+		}
+		types.ChrootMutex = &chrootExec.mu
+		exec = chrootExec
+		logging.Verbosef("server configured with chroot: %s", daemonConfig.ChrootDir)
+	}
+
+	return newCNIServer(daemonConfig.SocketDir, kubeClient, exec, serverConfig)
+}
+
+func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {
+
+	// preprocess server config to be used to override multus CNI config
+	// see extractCniData() for the detail
+	if servConfig != nil {
+		servConfig = bytes.Replace(servConfig, []byte("{"), []byte(","), 1)
+	}
+
+	router := http.NewServeMux()
+	s := &Server{
+		Server: http.Server{
+			Handler: router,
+		},
+		rundir:       rundir,
+		kubeclient:   kubeClient,
+		exec:         exec,
+		serverConfig: servConfig,
+		metrics: &Metrics{
+			requestCounter: prometheus.NewCounterVec(
+				prometheus.CounterOpts{
+					Name: "multus_server_request_total",
+					Help: "Counter of HTTP requests",
+				},
+				[]string{"handler", "code", "method"},
+			),
+		},
+	}
+	// register metrics
+	prometheus.MustRegister(s.metrics.requestCounter)
+
+	// handle for '/cni'
+	router.HandleFunc(api.MultusCNIAPIEndpoint, promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": api.MultusCNIAPIEndpoint}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != http.MethodPost {
+				http.Error(w, fmt.Sprintf("Method not allowed"), http.StatusMethodNotAllowed)
+				return
+			}
+
+			result, err := s.handleCNIRequest(r)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				return
+			}
+
+			w.WriteHeader(http.StatusOK)
+			// Empty response JSON means success with no body
+			w.Header().Set("Content-Type", "application/json")
+			if _, err := w.Write(result); err != nil {
+				_ = logging.Errorf("Error writing HTTP response: %v", err)
+			}
+		})))
+
+	// handle for '/delegate'
+	router.HandleFunc(api.MultusDelegateAPIEndpoint, promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": api.MultusDelegateAPIEndpoint}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != http.MethodPost {
+				http.Error(w, fmt.Sprintf("Method not allowed"), http.StatusMethodNotAllowed)
+				return
+			}
+
+			result, err := s.handleDelegateRequest(r)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				return
+			}
+
+			w.WriteHeader(http.StatusOK)
+			// Empty response JSON means success with no body
+			w.Header().Set("Content-Type", "application/json")
+			if _, err := w.Write(result); err != nil {
+				_ = logging.Errorf("Error writing HTTP response: %v", err)
+			}
+		})))
+
+	// handle for '/healthz'
+	router.HandleFunc(api.MultusHealthAPIEndpoint, promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": api.MultusHealthAPIEndpoint}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != http.MethodGet {
+				http.Error(w, fmt.Sprintf("Method not allowed"), http.StatusMethodNotAllowed)
+				return
+			}
+
+			w.WriteHeader(http.StatusOK)
+			w.Header().Set("Content-Type", "application/json")
+		})))
+
+	// this handle for the rest of above
+	router.HandleFunc("/", promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": "NotFound"}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_ = logging.Errorf("http not found: %v", r)
+			w.WriteHeader(http.StatusNotFound)
+		})))
+
+	return s, nil
+}
+
+func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
+	var cr api.Request
+	b, err := io.ReadAll(r.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &cr); err != nil {
+		return nil, err
+	}
+	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	if err != nil {
+		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
+	}
+
+	k8sArgs, err := kubernetesRuntimeArgs(cr.Env, s.kubeclient)
+	if err != nil {
+		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
+	}
+
+	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient)
+	if err != nil {
+		// Prefix error with request information for easier debugging
+		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+	}
+	return result, nil
+}
+
+func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
+	var cr api.Request
+	b, err := io.ReadAll(r.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &cr); err != nil {
+		return nil, err
+	}
+	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	if err != nil {
+		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
+	}
+
+	k8sArgs, err := kubernetesRuntimeArgs(cr.Env, s.kubeclient)
+	if err != nil {
+		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
+	}
+
+	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient, cr.InterfaceAttributes)
+	if err != nil {
+		// Prefix error with request information for easier debugging
+		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+	}
+	return result, nil
+}
+
+func extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
+	cmd, ok := cniRequest.Env["CNI_COMMAND"]
+	if !ok {
+		return "", nil, fmt.Errorf("unexpected or missing CNI_COMMAND")
+	}
+
+	cniCmdArgs := &skel.CmdArgs{}
+	cniCmdArgs.ContainerID, ok = cniRequest.Env["CNI_CONTAINERID"]
+	if !ok {
+		return "", nil, fmt.Errorf("missing CNI_CONTAINERID")
+	}
+	cniCmdArgs.Netns, ok = cniRequest.Env["CNI_NETNS"]
+	if !ok {
+		return "", nil, fmt.Errorf("missing CNI_NETNS")
+	}
+
+	cniCmdArgs.IfName, ok = cniRequest.Env["CNI_IFNAME"]
+	if !ok {
+		cniCmdArgs.IfName = "eth0"
+	}
+
+	cniArgs, found := cniRequest.Env["CNI_ARGS"]
+	if !found {
+		return "", nil, fmt.Errorf("missing CNI_ARGS")
+	}
+	cniCmdArgs.Args = cniArgs
+
+	if overrideConf != nil {
+		// trim the close bracket from multus CNI config and put the server config
+		// to override CNI config with server config.
+		// note: if there are two or more value in same key, then the
+		// latest one is used at golang json implementation
+		idx := bytes.LastIndex(cniRequest.Config, []byte("}"))
+		if idx == -1 {
+			return "", nil, fmt.Errorf("invalid CNI config")
+		}
+		cniCmdArgs.StdinData = append(cniRequest.Config[:idx], overrideConf...)
+	} else {
+		cniCmdArgs.StdinData = cniRequest.Config
+	}
+
+	return cmd, cniCmdArgs, nil
+}
+
+func kubernetesRuntimeArgs(cniRequestEnvVariables map[string]string, kubeClient *k8s.ClientInfo) (*types.K8sArgs, error) {
+	cniEnv, err := gatherCNIArgs(cniRequestEnvVariables)
+	if err != nil {
+		return nil, err
+	}
+	podNamespace, found := cniEnv["K8S_POD_NAMESPACE"]
+	if !found {
+		return nil, fmt.Errorf("missing K8S_POD_NAMESPACE")
+	}
+
+	podName, found := cniEnv["K8S_POD_NAME"]
+	if !found {
+		return nil, fmt.Errorf("missing K8S_POD_NAME")
+	}
+
+	uid, err := podUID(kubeClient, cniEnv, podNamespace, podName)
+	if err != nil {
+		return nil, err
+	}
+
+	sandboxID := cniRequestEnvVariables["K8S_POD_INFRA_CONTAINER_ID"]
+
+	return &types.K8sArgs{
+		K8S_POD_NAME:               cnitypes.UnmarshallableString(podName),
+		K8S_POD_NAMESPACE:          cnitypes.UnmarshallableString(podNamespace),
+		K8S_POD_INFRA_CONTAINER_ID: cnitypes.UnmarshallableString(sandboxID),
+		K8S_POD_UID:                cnitypes.UnmarshallableString(uid),
+	}, nil
+}
+
+func gatherCNIArgs(env map[string]string) (map[string]string, error) {
+	cniArgs, ok := env["CNI_ARGS"]
+	if !ok {
+		return nil, fmt.Errorf("missing CNI_ARGS: '%s'", env)
+	}
+
+	mapArgs := make(map[string]string)
+	for _, arg := range strings.Split(cniArgs, ";") {
+		parts := strings.Split(arg, "=")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid CNI_ARG '%s'", arg)
+		}
+		mapArgs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+	}
+	return mapArgs, nil
+}
+
+func podUID(kubeclient *k8s.ClientInfo, cniArgs map[string]string, podNamespace, podName string) (string, error) {
+	// UID may not be passed by all runtimes yet. Will be passed
+	// by CRIO 1.20+ and containerd 1.5+ soon.
+	// CRIO 1.20: https://github.com/cri-o/cri-o/pull/5029
+	// CRIO 1.21: https://github.com/cri-o/cri-o/pull/5028
+	// CRIO 1.22: https://github.com/cri-o/cri-o/pull/5026
+	// containerd 1.6: https://github.com/containerd/containerd/pull/5640
+	// containerd 1.5: https://github.com/containerd/containerd/pull/5643
+	uid, found := cniArgs["K8S_POD_UID"]
+	if !found {
+		pod, err := kubeclient.GetPod(podNamespace, podName)
+		if err != nil {
+			return "", fmt.Errorf("missing pod UID; attempted to recover it from the K8s API, but failed: %w", err)
+		}
+		return string(pod.UID), nil
+	}
+
+	return uid, nil
+}
+
+func cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return nil, fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	result, err := multus.CmdAdd(cmdArgs, exec, kubeClient)
+	if err != nil {
+		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
+	}
+	return serializeResult(result)
+}
+
+func cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdDel for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdDel(cmdArgs, exec, kubeClient)
+}
+
+func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdCheck for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdCheck(cmdArgs, exec, kubeClient)
+}
+
+func serializeResult(result cnitypes.Result) ([]byte, error) {
+	// cni result is converted to latest here and decoded to specific cni version at multus-shim
+	realResult, err := cni100.NewResultFromResult(result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate the CNI result: %w", err)
+	}
+
+	responseBytes, err := json.Marshal(&api.Response{Result: realResult})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal pod request response: %v", err)
+	}
+	return responseBytes, nil
+}
+
+func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return nil, fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// copy deleate annotation into network selection element
+	var selectionElement *types.NetworkSelectionElement
+	if interfaceAttributes != nil {
+		selectionElement = &types.NetworkSelectionElement{}
+		if interfaceAttributes.MacRequest != "" {
+			selectionElement.MacRequest = interfaceAttributes.MacRequest
+		}
+		if interfaceAttributes.IPRequest != nil {
+			selectionElement.IPRequest = interfaceAttributes.IPRequest
+		}
+		if interfaceAttributes.CNIArgs != nil {
+			selectionElement.CNIArgs = interfaceAttributes.CNIArgs
+		}
+	}
+
+	delegateCNIConf, err := types.LoadDelegateNetConf(cmdArgs.StdinData, selectionElement, "", "")
+	if err != nil {
+		return nil, err
+	}
+
+	logging.Debugf("CmdDelegateAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
+	result, err := multus.DelegateAdd(exec, kubeClient, pod, delegateCNIConf, rt, multusConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
+	}
+
+	return serializeResult(result)
+}
+
+func cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, _ *k8s.ClientInfo, multusConfig *types.NetConf) error {
+	delegateCNIConf := &types.DelegateNetConf{}
+	if err := json.Unmarshal(cmdArgs.StdinData, delegateCNIConf); err != nil {
+		return err
+	}
+	delegateCNIConf.Bytes = cmdArgs.StdinData
+	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
+	return multus.DelegateCheck(exec, delegateCNIConf, rt, multusConfig)
+}
+
+// note: this function may send back error to the client. In cni spec, command DEL should NOT send any error
+// because container deletion follows cni DEL command. But in delegateDel case, container is not removed by
+// this delegateDel, hence we decide to send error message to the request sender.
+func cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	if err != nil {
+		return err
+	}
+
+	delegateCNIConf, err := types.LoadDelegateNetConf(cmdArgs.StdinData, nil, "", "")
+	if err != nil {
+		return err
+	}
+	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
+	return multus.DelegateDel(exec, pod, delegateCNIConf, rt, multusConfig)
+}
+
+// LoadDaemonNetConf loads the configuration for the multus daemon
+func LoadDaemonNetConf(config []byte) (*ControllerNetConf, error) {
+	daemonNetConf := &ControllerNetConf{
+		SocketDir: DefaultMultusRunDir,
+	}
+	if err := json.Unmarshal(config, daemonNetConf); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the daemon configuration: %w", err)
+	}
+
+	logging.SetLogStderr(daemonNetConf.LogToStderr)
+	if daemonNetConf.LogFile != DefaultMultusDaemonConfigFile {
+		logging.SetLogFile(daemonNetConf.LogFile)
+	}
+	if daemonNetConf.LogLevel != "" {
+		logging.SetLogLevel(daemonNetConf.LogLevel)
+	}
+	daemonNetConf.ConfigFileContents = config
+
+	return daemonNetConf, nil
+}