Merge pull request #924 from k8snetworkplumbingwg/fix/image

Fix release 3.x thick image tag to isolate from 4.0

.github/workflows/build.yml

@@ -4,18 +4,18 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.22.x, 1.23.x]
+        go-version: [1.17.x, 1.18.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v2
 
     - name: Build
       env:

.github/workflows/codeql.yml

@@ -1,41 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "master" ]
-  pull_request:
-    branches: [ "master" ]
-  schedule:
-    - cron: "46 8 * * 0"
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ go ]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          queries: +security-and-quality
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
-        with:
-          category: "/language:${{ matrix.language }}"

.github/workflows/image-build.yml

@@ -1,98 +1,128 @@
 name: Image build
 on: [pull_request]
 jobs:
-  build-thin:
-    name: Image build thin plugin
+  ep-build-amd64:
+    name: Image build/amd64 LEGACY entrypoint
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
 
-      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:latest
+          tags: ghcr.io/${{ github.repository }}:ep-latest-amd64
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
 
-      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
-      - name: Build container debug image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: false
-          tags: ghcr.io/${{ github.repository }}:latest
-          file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
-
-  build-thick:
-    name: Image thick plugin
+  build-amd64:
+    name: Image build/amd64 daemonized alternative
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
 
       - name: Build container image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:latest-thick
+          tags: ghcr.io/${{ github.repository }}:latest-amd64
           file: images/Dockerfile.thick
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.29.0
+  build-arm64:
+    name: Image build/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
         with:
-          image-ref: ghcr.io/${{ github.repository }}:latest-thick
-          ignore-unfixed: true
-          vuln-type: 'os,library'
-          severity: 'CRITICAL,HIGH'
-          format: 'sarif'
-          output: 'trivy-results.sarif'
-          
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        if: always()
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm64
+          file: images/Dockerfile.arm64
+
+  build-arm32:
+    name: Image build/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
         with:
-          sarif_file: 'trivy-results.sarif'
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm32
+          file: images/Dockerfile.arm32
+
+  build-ppc64le:
+    name: Image build/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  build-s390:
+    name: Image build/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-s390x
+          file: images/Dockerfile.s390x
 
   build-origin:
     name: Image build/origin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
 
-      - name: Download OKD Builder Dockerfile
-        run: curl https://raw.githubusercontent.com/okd-project/images/main/builder/Dockerfile -o images/okd-builder.Dockerfile
-
-      - name: Patch OKD Builder Dockerfile to workaround error
-        run: sed -i -e "s/yum install -y yum-utils/rpm --import \/etc\/pki\/rpm-gpg\/*;yum install -y yum-utils/" images/okd-builder.Dockerfile
-
-      - name: Create root for builder
-        run: mkdir root
-
-      - name: Organically build golang builder image
-        run: docker build -t local/okdbuilder:latest -f images/okd-builder.Dockerfile .
-
-      - name: Organically build Multus origin image
-        run: docker build -t local/multus-cni:latest-origin -f images/Dockerfile.openshift .
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-origin
+          file: images/Dockerfile.openshift

.github/workflows/image-push-master.yml

@@ -1,116 +1,240 @@
 name: Image push for master
-on:
+on: 
   push:
     branches:
       - master
-env:
-  image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick:
-    name: Image push thick image
+  push-amd64:
+    name: Image push/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Container Registry
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v3
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Push container image for thick plugin
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:latest-thick
-            ghcr.io/${{ github.repository }}:snapshot-thick
-          file: images/Dockerfile.thick
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
-
-  push-thin:
-    name: Image push thin image
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push thin container image
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:latest
-            ghcr.io/${{ github.repository }}:snapshot
+            ghcr.io/${{ github.repository }}:latest-amd64
+            ghcr.io/${{ github.repository }}:snapshot-amd64
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
 
-      - name: Push thin container debug image
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:latest-debug
-            ghcr.io/${{ github.repository }}:snapshot-debug
-          file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
+            ghcr.io/${{ github.repository }}:thick-amd64
+          file: images/Dockerfile.thick
 
-# TODO: need to fix this action
-#  push-origin:
-#    name: Image push/origin
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v4
-#
-#      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v3
-#
-#      - name: Login to GitHub Container Registry
-#        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v3
-#        with:
-#          registry: ghcr.io
-#          username: ${{ github.repository_owner }}
-#          password: ${{ secrets.GITHUB_TOKEN }}
-#
-#      - name: Push container image
-#        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v5
-#        with:
-#          context: .
-#          push: true
-#          tags: |
-#            ghcr.io/${{ github.repository }}:latest-origin
-#            ghcr.io/${{ github.repository }}:snapshot-origin
-#          file: images/Dockerfile.openshift
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-arm64
+            ghcr.io/${{ github.repository }}:snapshot-arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-arm32
+            ghcr.io/${{ github.repository }}:snapshot-arm32
+          file: images/Dockerfile.arm32
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-ppc64le
+            ghcr.io/${{ github.repository }}:snapshot-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-s390x
+            ghcr.io/${{ github.repository }}:snapshot-s390x
+          file: images/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-origin
+            ghcr.io/${{ github.repository }}:snapshot-origin
+          file: images/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest create ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64 --arch amd64
+          docker manifest push ${{ env.REPOSITORY }}:thick
+
+          docker pull ${{ env.REPOSITORY }}:snapshot-amd64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm32
+          docker pull ${{ env.REPOSITORY }}:snapshot-ppc64le
+          docker pull ${{ env.REPOSITORY }}:snapshot-s390x
+          docker pull ${{ env.REPOSITORY }}:latest-amd64
+          docker pull ${{ env.REPOSITORY }}:latest-arm64
+          docker pull ${{ env.REPOSITORY }}:latest-arm32
+          docker pull ${{ env.REPOSITORY }}:latest-ppc64le
+          docker pull ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-arm32 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:snapshot
+          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-arm32 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:latest

.github/workflows/image-push-release.yml

@@ -1,24 +1,22 @@
 name: Image push release
-on:
+on: 
   push:
     tags:
       - v*
-env:
-  image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick:
-    name: Image push thick image
+  push-amd64:
+    name: Image push/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Container Registry
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v3
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -26,113 +24,262 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v5
+        uses: crazy-max/ghaction-docker-meta@v1
         with:
           images: ghcr.io/${{ github.repository }}
-          flavor: |
-            latest=false
+          tag-latest: false
 
-      - name: Push container image for thick plugin
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:stable-thick
-            ${{ steps.docker_meta.outputs.tags }}-thick
-          file: images/Dockerfile.thick
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
-
-  push-thin:
-    name: Image push thin image/amd64
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ghcr.io/${{ github.repository }}
-          flavor: |
-            latest=false
-
-      - name: Push thin container image
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}:stable
-            ${{ steps.docker_meta.outputs.tags }}
+            ghcr.io/${{ github.repository }}:stable-amd64
+            ${{ steps.docker_meta.outputs.tags }}-amd64
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
 
-      - name: Push thin container debug image
-        if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v5
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}:stable-debug
-            ${{ steps.docker_meta.outputs.tags }}-debug
-          file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
-          sbom: false
-          provenance: false
+            ghcr.io/${{ github.repository }}:stable-thick-amd64
+            ${{ steps.docker_meta.outputs.tags }}-thick-amd64
+          file: images/Dockerfile.thick
 
-# TODO: need to fix this action
-#  push-origin:
-#    name: Image push/origin
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v4
-#
-#      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v3
-#
-#      - name: Login to GitHub Container Registry
-#        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v3
-#        with:
-#          registry: ghcr.io
-#          username: ${{ github.repository_owner }}
-#          password: ${{ secrets.GITHUB_TOKEN }}
-#
-#      - name: Docker meta
-#        id: docker_meta
-#        uses: crazy-max/ghaction-docker-meta@v1
-#        with:
-#          images: ghcr.io/${{ github.repository }}
-#          tag-latest: false
-#
-#      - name: Push container image
-#        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v5
-#        with:
-#          context: .
-#          push: true
-#          tags: |
-#            ghcr.io/${{ github.repository }}:stable-origin
-#            ${{ steps.docker_meta.outputs.tags }}-origin
-#          file: images/Dockerfile.openshift
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-arm64
+            ${{ steps.docker_meta.outputs.tags }}-arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-arm32
+            ${{ steps.docker_meta.outputs.tags }}-arm32
+          file: images/Dockerfile.arm32
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-ppc64le
+            ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-s390x
+            ${{ steps.docker_meta.outputs.tags }}-s390x
+          file: images/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-origin
+            ${{ steps.docker_meta.outputs.tags }}-origin
+          file: images/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ steps.docker_meta.outputs.tags }}-amd64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm32
+          docker pull ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          docker pull ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-arm32 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 --arch amd64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm64 --arch arm64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm32 --arch arm
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-ppc64le --arch ppc64le
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-s390x --arch s390x
+          docker manifest push ${{ steps.docker_meta.outputs.tags }}
+          docker pull ${{ env.REPOSITORY }}:stable-amd64
+          docker pull ${{ env.REPOSITORY }}:stable-arm64
+          docker pull ${{ env.REPOSITORY }}:stable-arm32
+          docker pull ${{ env.REPOSITORY }}:stable-ppc64le
+          docker pull ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-arm32 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:stable

.github/workflows/kind-e2e.yml

@@ -3,71 +3,29 @@ on: [push, pull_request]
 jobs:
   e2e-kind:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - docker-file: images/Dockerfile.thick
-            cni-version: "0.3.1"
-            multus-manifest: multus-daemonset-thick.yml
-          - docker-file: images/Dockerfile
-            cni-version: "0.3.1"
-            multus-manifest: multus-daemonset.yml
-          - docker-file: images/Dockerfile.thick
-            cni-version: "0.4.0"
-            multus-manifest: multus-daemonset-thick.yml
-          - docker-file: images/Dockerfile
-            cni-version: "0.4.0"
-            multus-manifest: multus-daemonset.yml
-          # need to wait kind to support CNI 1.0.0 (now kind 0.11 supports up to 0.4.0)
-#          - docker-file: images/Dockerfile.thick
-#            cni-version: "1.0.0"
-#            multus-manifest: multus-thick-daemonset.yml
-#          - docker-file: images/Dockerfile
-#            cni-version: "1.0.0"
-#            multus-manifest: multus-daemonset.yml
-    env:
-      JOB_NAME: "${{ matrix.cni-version }}-${{ matrix.multus-manifest }}"
-
     if: >
       (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
         github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.x
-
-      - name: Setup j2cli
-        run: |
-          sudo apt-get install -y j2cli
-          echo $(j2 --version)
+        uses: actions/checkout@v2
 
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+  
       - name: Build latest-amd64
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          load: true
-          tags: localhost:5000/multus:e2e
-          file: ${{ matrix.docker-file }}
-          platforms: linux/amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile.thick .
+ 
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
 
       - name: Get kind/kubectl/koko
         working-directory: ./e2e
         run: ./get_tools.sh
 
-      - name: generate yaml files
-        working-directory: ./e2e
-        run: env CNI_VERSION=${{ matrix.cni-version }} ./generate_yamls.sh
-
       - name: Setup cluster
         working-directory: ./e2e
-        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} MULTUS_DOCKERFILE=none ./setup_cluster.sh
+        run: ./setup_cluster.sh
 
       - name: Test simple pod
         working-directory: ./e2e
@@ -85,33 +43,8 @@ jobs:
         working-directory: ./e2e
         run: ./test-default-route1.sh
 
-#      - name: Test DRA integration
-#        working-directory: ./e2e
-#        run: ./test-dra-integration.sh
-
-      - name: Test subdirectory CNI chaining
-        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
-        working-directory: ./e2e
-        run: ./test-subdirectory-chaining.sh
-
-      - name: Test subdirectory CNI chaining with passthru CNI / auxiliaryCNIChainName
-        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
-        working-directory: ./e2e
-        run: ./test-subdirectory-chaining-passthru.sh
-
-      - name: Export kind logs
-        if: always()
-        run: |
-          mkdir -p /tmp/kind/logs
-          kind export logs /tmp/kind/logs -v 2147483647
-
-      - name: Upload kind logs
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
-          path: /tmp/kind/logs
-
       - name: cleanup cluster and registry
         run: |
           kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/legacy-kind-e2e.yml

@@ -0,0 +1,46 @@
+name: e2e-kind legacy installation with entrypoint script
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile .
+
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: MULTUS_MANIFEST=legacy-multus-daemonset.yml ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/release.yml

@@ -8,17 +8,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v2
         with:
-          go-version: 1.22.x
+          go-version: 1.17.x
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v2
         with:
           version: latest
           args: release --rm-dist

.github/workflows/stale-issues-prs.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@v3
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'

.github/workflows/test.yml

@@ -4,17 +4,17 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.22.x, 1.23.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v2
 
     - name: Run Revive Action by pulling pre-built image
       uses: docker://morphy/revive-action:v2

.gitignore

@@ -1,8 +1,6 @@
 # Binary output dir
 bin/
 e2e/bin/
-e2e/yamls/
-e2e/repos/
 
 # GOPATH created by the build script
 gopath/

.goreleaser.yml

@@ -6,51 +6,20 @@ before:
   hooks:
     - go mod download
 builds:
-  - env:
-      - CGO_ENABLED=0
-    id: multus
-    binary: multus
-    main: ./cmd/multus
-    goos:
-      - linux
-    goarch:
-      - 386
-      - amd64
-      - arm
-      - arm64
-      - s390x
-    ldflags:
-      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
-  - env:
-      - CGO_ENABLED=0
-    id: multus-daemon
-    binary: multus-daemon
-    main: ./cmd/multus-daemon
-    goos:
-      - linux
-    goarch:
-      - 386
-      - amd64
-      - arm
-      - arm64
-      - s390x
-    ldflags:
-      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
-  - env:
-      - CGO_ENABLED=0
-    id: multus-shim
-    binary: multus-shim
-    main: ./cmd/multus-shim
-    goos:
-      - linux
-    goarch:
-      - 386
-      - amd64
-      - arm
-      - arm64
-      - s390x
-    ldflags:
-      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+-  
+  env:
+  - CGO_ENABLED=0
+  main: ./cmd/
+  goos:
+  - linux
+  goarch:
+  - 386
+  - amd64
+  - arm
+  - arm64
+  - s390x
+  ldflags:
+  - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date={{ .Date }}
 archives:
   - wrap_in_directory: true
 checksum:

.travis.yml

@@ -0,0 +1,116 @@
+os: linux
+language: go
+# see https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments
+# for the detail
+# sudo: requried
+dist: bionic
+
+services:
+  - docker
+
+go:
+  - 1.13.x
+
+env:
+  global:
+    - GO111MODULE=on
+    - REGISTRY_USER=${REGISTRY_USER:-nfvpe}
+    - REGISTRY_PASS=${REGISTRY_PASS}
+    - REPOSITORY_NAME=${REPOSITORY_NAME}
+    - REPOSITORY_USER=${REPOSITORY_USER}
+    - DOCKER_CLI_EXPERIMENTAL="enabled"
+    - secure: "${REGISTRY_SECURE}"
+  jobs:
+    - TARGET=amd64
+    - TARGET=ppc64le
+
+before_install:
+  - if [ "${REPOSITORY_NAME}" = "" ]; then export REPOSITORY_NAME=multus; fi
+  - sudo apt-get update -qq
+  - go get github.com/mattn/goveralls
+
+install:
+  - go get -u golang.org/x/lint/golint
+
+before_script:
+  # Make gopath... to run golint/go fmt/go vet
+  # Suppress golint for fixing lint later.
+  - golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
+  - go fmt ./...
+  - go vet ./...
+#  - gocyclo -over 15 ./multus
+
+script:
+  - GOARCH="${TARGET}" ./hack/build-go.sh
+  - |
+    if [ "${TARGET}" == "amd64" ]; then
+      sudo env PATH=${PATH} ./scripts/test.sh
+      goveralls -coverprofile=coverage.out -service=travis-ci
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 .
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le -f Dockerfile.ppc64le .
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}-origin:latest -f Dockerfile.openshift .
+    fi
+
+deploy:
+  # Release on versioned tag (e.g. v1.0)
+  - provider: script
+    #cleanup: false
+    script: curl -sL https://git.io/goreleaser
+    on:
+      tags: true
+      all_branches: true
+      condition: "$TARGET = amd64 && $TRAVIS_TAG =~ ^v[0-9].*$ && ! -z $GITHUB_TOKEN && $TRAVIS_OS_NAME = linux"
+  # Push images to Dockerhub on tag
+  - provider: script
+    cleanup: false
+    script: >
+      bash -c '
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
+      export DOCKER_CLI_EXPERIMENTAL="enabled";
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
+      echo done'
+    on:
+      tags: true
+      all_branches: true
+      condition: "$TRAVIS_TAG =~ ^v[0-9].*$ && -n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
+  # Push images to Dockerhub on merge to master
+  - provider: script
+    on:
+      branch: master
+      condition: "-n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
+    script: >
+      bash -c '
+      docker tag ${REPOSITORY_USER}/:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      echo done'

Makefile

@@ -1,14 +0,0 @@
-
-
-.PHONY: deps-update
-deps-update: ; $(info  Updating dependencies...) @ ## Update dependencies
-	go mod tidy
-	go mod vendor
-
-PHONY: build test
-
-build:
-	./hack/build-go.sh
-
-test:
-	sudo ./hack/test-go.sh

NOTICE

@@ -1,2 +0,0 @@
-Copyright 2016 Intel Corporation
-Copyright 2021 Multus Authors

README.md

@@ -22,45 +22,30 @@ Here's an illustration of the network interfaces attached to a pod, as provision
 
 ## Quickstart Installation Guide
 
-The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugin, refer to our [quick-start guide](docs/quickstart.md).
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
 
-To use latest features try command below which applies a daemonset and installs thick Multus using `kubectl`:
+Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
 
 ```
-kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
 
-## Thin Plugin v.s Thick Plugin
-
-With the multus 4.0 release, we introduce a new client/server-style plugin deployment. This new deployment is called ['thick plugin'](docs/thick-plugin.md), in contrast to deployments in previous versions, which is now called a 'thin plugin'. The new thick plugin consists of two binaries, multus-daemon and multus-shim CNI plugin. The 'multus-daemon' will be deployed to all nodes as a local agent and supports additional features, such as metrics, which were not available with the 'thin plugin' deployment before. Due to these additional features, the 'thick plugin' comes with the trade-off of consuming more resources than the 'thin plugin'.
-
-We recommend using the thick plugin in most environments, but if you wish to run the thin plugin, or are in a resource-constrained environment, you may do so with:
-
-```
-kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
-```
-
-## Additional Installation Options
-
-In addition to the [quick-start guide](docs/quickstart.md), you may:
+## Additional installation Options
 
+- Install via daemonset using the quick-start guide, above.
 - Download binaries from [release page](https://github.com/k8snetworkplumbingwg/multus-cni/releases)
-- By Docker image from [GitHub Container Registry](https://github.com/orgs/k8snetworkplumbingwg/packages/container/package/multus-cni)
+- By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
 - Or, roll-your-own and build from source
   - See [Development](docs/development.md)
 
 ## Comprehensive Documentation
 
 - [How to use](docs/how-to-use.md)
-- [Quick Start Guide](docs/quickstart.md)
 - [Configuration](docs/configuration.md)
-- [Development and Support Information](docs/development.md)
-- [Thick Plugin](docs/thick-plugin.md)
+- [Development](docs/development.md)
 
 ## Contact Us
 
-For any questions about Multus CNI, open up a GitHub issue or feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/).
-
-To be invited, use [this slack invite link](https://join.slack.com/t/npwg-team/shared_invite/zt-1u2vmsn2b-tKdOokdPY73zn9B32JoAOg).
+For any questions about Multus CNI, feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/), or open up a GitHub issue. Request an invite to NPWG slack [here](https://intel-corp.herokuapp.com/).

cmd/cert-approver/main.go

@@ -1,371 +0,0 @@
-// Copyright (c) 2023 Network Plumbing Working Group
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is Kubernetes controller which approves CSR submitted by multus.
-// This command is required only if multus runs with per-node certificate.
-package main
-
-// Note: cert-approver should be simple, just approve multus' CSR, hence
-// this go code should not have any dependencies from pkg/, if possible,
-// to keep its code simplicity.
-import (
-	"context"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-	"os"
-	"os/signal"
-	"reflect"
-	"strings"
-	"syscall"
-	"time"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/types"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/validation"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	certificatesv1 "k8s.io/api/certificates/v1"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/klog/v2"
-
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/client-go/util/certificate/csr"
-	"k8s.io/client-go/util/workqueue"
-)
-
-// CertController object
-type CertController struct {
-	clientset          kubernetes.Interface
-	queue              workqueue.RateLimitingInterface
-	informer           cache.SharedIndexInformer
-	broadcaster        record.EventBroadcaster
-	recorder           record.EventRecorder
-	commonNamePrefixes string
-}
-
-const (
-	maxDuration                = time.Hour * 24 * 365
-	resyncPeriod time.Duration = time.Second * 3600 // resync every one hour, default is 10 hour
-	maxRetries                 = 5
-)
-
-var (
-	// ControllerName provides controller name
-	ControllerName = "csr-approver"
-	// NamePrefix specifies which name in certification request should be target to approve
-	NamePrefix = "system:multus"
-	// Organization specifies which org in certification request should be target to approve
-	Organization = []string{"system:multus"}
-	// Groups specifies which group in certification request should be target to approve
-	Groups = sets.New[string]("system:nodes", "system:multus", "system:authenticated")
-	// UserPrefixes specifies which name prefix in certification request should be target to approve
-	UserPrefixes = sets.New[string]("system:node", NamePrefix)
-	// Usages specifies which usage in certification request should be target to approve
-	Usages = sets.New[certificatesv1.KeyUsage](
-		certificatesv1.UsageDigitalSignature,
-		certificatesv1.UsageClientAuth)
-)
-
-// NewCertController creates certcontroller
-func NewCertController() (*CertController, error) {
-	var clientset kubernetes.Interface
-	// setup Kubernetes API client
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	clientset, err = kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	informer := cache.NewSharedIndexInformer(
-		cache.NewListWatchFromClient(
-			clientset.CertificatesV1().RESTClient(),
-			"certificatesigningrequests", corev1.NamespaceAll, fields.Everything()),
-		&certificatesv1.CertificateSigningRequest{},
-		resyncPeriod,
-		nil)
-
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(klog.Infof)
-	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientset.CoreV1().Events("")})
-	recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "cert-approver"})
-	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
-	c := &CertController{
-		clientset:          clientset,
-		informer:           informer,
-		queue:              queue,
-		commonNamePrefixes: NamePrefix,
-		broadcaster:        broadcaster,
-		recorder:           recorder,
-	}
-
-	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc: func(obj interface{}) {
-			if csr, ok := obj.(*certificatesv1.CertificateSigningRequest); ok {
-				if c.filterCSR(csr) {
-					key, err := cache.MetaNamespaceKeyFunc(obj)
-					if err == nil {
-						queue.Add(key)
-					}
-				}
-			}
-		},
-	})
-
-	return c, nil
-}
-
-// Run starts controller
-func (c *CertController) Run(stopCh <-chan struct{}) {
-	defer utilruntime.HandleCrash()
-	defer c.queue.ShutDown()
-
-	klog.Info("Starting cert approver")
-
-	go c.informer.Run(stopCh)
-	if !cache.WaitForCacheSync(stopCh, c.HasSynced) {
-		utilruntime.HandleError(fmt.Errorf("Timed out waiting for caches to sync"))
-		return
-	}
-
-	klog.Info("cert approver synced and ready")
-	wait.Until(c.runWorker, time.Second, stopCh)
-}
-
-// HasSynced is required for the cache.Controller interface.
-func (c *CertController) HasSynced() bool {
-	return c.informer.HasSynced()
-}
-
-// LastSyncResourceVersion is required for the cache.Controller interface.
-func (c *CertController) LastSyncResourceVersion() string {
-	return c.informer.LastSyncResourceVersion()
-}
-
-func (c *CertController) runWorker() {
-	for c.processNextItem() {
-		// continue looping
-	}
-}
-
-func (c *CertController) processNextItem() bool {
-	// Wait until there is a new item in the working queue
-	key, quit := c.queue.Get()
-	if quit {
-		return false
-	}
-	// Tell the queue that we are done with processing this key. This unblocks the key for other workers
-	// This allows safe parallel processing because two pods with the same key are never processed in
-	// parallel.
-	defer c.queue.Done(key)
-
-	// Invoke the method containing the business logic
-	err := c.processItem(key.(string))
-	// Handle the error if something went wrong during the execution of the business logic
-	c.handleErr(err, key)
-	return true
-
-}
-
-// handleErr checks if an error happened and makes sure we will retry later.
-func (c *CertController) handleErr(err error, key interface{}) {
-	if err == nil {
-		// Forget about the #AddRateLimited history of the key on every successful synchronization.
-		// This ensures that future processing of updates for this key is not delayed because of
-		// an outdated error history.
-		c.queue.Forget(key)
-		return
-	}
-
-	// This controller retries 5 times if something goes wrong. After that, it stops trying.
-	if c.queue.NumRequeues(key) < maxRetries {
-		klog.Infof("Error syncing csr %s: %v", key, err)
-		// Re-enqueue the key rate limited. Based on the rate limiter on the
-		// queue and the re-enqueue history, the key will be processed later again.
-		c.queue.AddRateLimited(key)
-		return
-	}
-
-	c.queue.Forget(key)
-	// Report to an external entity that, even after several retries, we could not successfully process this key
-	utilruntime.HandleError(err)
-	klog.Infof("Dropping csr %q out of the queue: %v", key, err)
-}
-
-func (c *CertController) processItem(key string) error {
-	startTime := time.Now()
-
-	obj, _, err := c.informer.GetIndexer().GetByKey(key)
-	if err != nil {
-		return fmt.Errorf("Error fetching object with key %s from store: %v", key, err)
-	}
-
-	req, _ := obj.(*certificatesv1.CertificateSigningRequest)
-
-	nodeName := "unknown"
-	defer func() {
-		klog.Infof("Finished syncing CSR %s for %s node in %v", req.Name, nodeName, time.Since(startTime))
-	}()
-
-	if len(req.Status.Certificate) > 0 {
-		klog.V(5).Infof("CSR %s is already signed", req.Name)
-		return nil
-	}
-
-	if isApprovedOrDenied(&req.Status) {
-		klog.V(5).Infof("CSR %s is already approved/denied", req.Name)
-		return nil
-	}
-
-	csrPEM, _ := pem.Decode(req.Spec.Request)
-	if csrPEM == nil {
-		return fmt.Errorf("failed to PEM-parse the CSR block in .spec.request: no CSRs were found")
-	}
-
-	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
-	if err != nil {
-		return fmt.Errorf("failed to parse the CSR bytes: %v", err)
-	}
-
-	i := strings.LastIndex(req.Spec.Username, ":")
-	if i == -1 || i == len(req.Spec.Username)-1 {
-		return fmt.Errorf("failed to parse the username: %s", req.Spec.Username)
-	}
-
-	ctx := context.Background()
-	prefix := req.Spec.Username[:i]
-	nodeName = req.Spec.Username[i+1:]
-	if !UserPrefixes.Has(prefix) {
-		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by an unexpected user: %q", req.Name, req.Spec.Username))
-	}
-
-	if errs := validation.IsDNS1123Subdomain(nodeName); len(errs) != 0 {
-		return c.denyCSR(ctx, req, fmt.Sprintf("extracted node name %q is not a valid DNS subdomain %v", nodeName, errs))
-	}
-
-	if usages := sets.New[certificatesv1.KeyUsage](req.Spec.Usages...); !usages.Equal(Usages) {
-		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with unexpected usages: %v", req.Name, usages.UnsortedList()))
-	}
-
-	if !Groups.HasAll(req.Spec.Groups...) {
-		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by a user with unexpected groups: %v", req.Name, req.Spec.Groups))
-	}
-
-	expectedSubject := fmt.Sprintf("%s:%s", c.commonNamePrefixes, nodeName)
-	if x509CSR.Subject.CommonName != expectedSubject {
-		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's commonName to be %q, but it is %q", expectedSubject, x509CSR.Subject.CommonName))
-	}
-
-	if !reflect.DeepEqual(x509CSR.Subject.Organization, Organization) {
-		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's organization to be %v, but it is %v", Organization, x509CSR.Subject.Organization))
-	}
-
-	if req.Spec.ExpirationSeconds == nil {
-		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created without specyfying the expirationSeconds", req.Name))
-	}
-
-	if csr.ExpirationSecondsToDuration(*req.Spec.ExpirationSeconds) > maxDuration {
-		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with invalid expirationSeconds value: %d", req.Name, *req.Spec.ExpirationSeconds))
-	}
-
-	return c.approveCSR(ctx, req)
-}
-
-// CSR specific functions
-
-func (c *CertController) filterCSR(csr *certificatesv1.CertificateSigningRequest) bool {
-	nsName := types.NamespacedName{Namespace: csr.Namespace, Name: csr.Name}
-	csrPEM, _ := pem.Decode(csr.Spec.Request)
-	if csrPEM == nil {
-		klog.Errorf("Failed to PEM-parse the CSR block in .spec.request: no CSRs were found in %s", nsName)
-		return false
-	}
-
-	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
-	if err != nil {
-		klog.Errorf("Failed to parse the CSR .spec.request of %q: %v", nsName, err)
-		return false
-	}
-
-	return strings.HasPrefix(x509CSR.Subject.CommonName, c.commonNamePrefixes) &&
-		csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName
-}
-
-func (c *CertController) approveCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error {
-	csr.Status.Conditions = append(csr.Status.Conditions,
-		certificatesv1.CertificateSigningRequestCondition{
-			Type:    certificatesv1.CertificateApproved,
-			Status:  corev1.ConditionTrue,
-			Reason:  "AutoApproved",
-			Message: fmt.Sprintf("Auto-approved CSR %q", csr.Name),
-		})
-
-	c.recorder.Eventf(csr, corev1.EventTypeNormal, "CSRApproved", "CSR %q has been approved by %s", csr.Name, ControllerName)
-	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
-	return err
-}
-
-func (c *CertController) denyCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, message string) error {
-	csr.Status.Conditions = append(csr.Status.Conditions,
-		certificatesv1.CertificateSigningRequestCondition{
-			Type:    certificatesv1.CertificateDenied,
-			Status:  corev1.ConditionTrue,
-			Reason:  "CSRDenied",
-			Message: message,
-		},
-	)
-
-	c.recorder.Eventf(csr, corev1.EventTypeWarning, "CSRDenied", "The CSR %q has been denied by: %s", csr.Name, ControllerName, message)
-	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().Update(ctx, csr, metav1.UpdateOptions{})
-	return err
-}
-
-func isApprovedOrDenied(status *certificatesv1.CertificateSigningRequestStatus) bool {
-	for _, c := range status.Conditions {
-		if c.Type == certificatesv1.CertificateApproved || c.Type == certificatesv1.CertificateDenied {
-			return true
-		}
-	}
-	return false
-}
-
-func main() {
-	klog.Infof("starting cert-approver")
-
-	// Start watching for pod creations
-	certController, err := NewCertController()
-	if err != nil {
-		klog.Fatal(err)
-	}
-
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-	go certController.Run(stopCh)
-
-	sigterm := make(chan os.Signal, 1)
-	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
-	<-sigterm
-}

cmd/config-generation/kubeconfig.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this generates kubeconfig file for multus based on service account
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+const userRWPermission = 0600
+
+const (
+	cniConfigDirVarName   = "cni-config-dir"
+	k8sCAFilePathVarName  = "kube-ca-file"
+	k8sServiceHostVarName = "k8s-service-host"
+	k8sServicePortVarName = "k8s-service-port"
+	serviceAccountPath    = "/var/run/secrets/kubernetes.io/serviceaccount"
+	skipTLSVerifyVarName  = "skip-tls-verify"
+)
+
+const (
+	defaultCniConfigDir   = "/host/etc/cni/net.d"
+	defaultK8sCAFilePath  = ""
+	defaultK8sServiceHost = ""
+	defaultK8sServicePort = 0
+	defaultSkipTLSValue   = false
+)
+
+func main() {
+	k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
+	k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
+	skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
+	kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	flag.Parse()
+
+	if *k8sServiceHost == defaultK8sServiceHost {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *k8sServicePort == defaultK8sServicePort {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *kubeCAFilePath == defaultK8sServiceHost {
+		*kubeCAFilePath = serviceAccountPath + "/ca.crt"
+	}
+
+	tlsCfg := "insecure-skip-tls-verify: true"
+	if !*skipTLSVerify {
+		kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
+		if err != nil {
+			logError("failed grabbing CA file: %w", err)
+		}
+		tlsCfg = "certificate-authority-data: " + kubeCAFileContents
+	}
+
+	multusConfigDir := *cniConfigDir + "/multus.d/"
+	if err := prepareCNIConfigDir(multusConfigDir); err != nil {
+		logError("failed to create CNI config dir: %w", err)
+	}
+	kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
+	serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
+	if err != nil {
+		logError("failed grabbing k8s token: %w", err)
+	}
+	if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
+		logError("failed generating kubeconfig: %w", err)
+	}
+}
+
+func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
+	data, err := ioutil.ReadFile(pathCAFile)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
+	}
+	return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
+}
+
+func k8sKubeConfigToken(tokenPath string) (string, error) {
+	data, err := ioutil.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
+	}
+	return string(data), nil
+}
+
+func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
+	kubeConfigTemplate := `
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: %s://[%s]:%d
+    %s
+users:
+- name: multus
+  user:
+    token: "%s"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+	kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
+	logInfo("Generated KubeConfig saved to %s: \n%s", outputPath, kubeconfig)
+	return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
+}
+
+func prepareCNIConfigDir(cniConfigDirPath string) error {
+	return os.MkdirAll(cniConfigDirPath, userRWPermission)
+}
+
+func logInvalidArg(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func logError(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	os.Exit(1)
+}
+
+func logInfo(format string, values ...interface{}) {
+	log.Printf("INFO: %s", fmt.Sprintf(format, values...))
+}

cmd/controller/main.go

@@ -0,0 +1,253 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this is daemonized entrypoint process. which watches master config
+// and generate multus CNI config
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+const (
+	multusPluginName     = "multus"
+	multusConfigFileName = "00-multus.conf"
+)
+
+const (
+	defaultCniConfigDir                 = "/etc/cni/net.d"
+	defaultMultusAdditionalBinDir       = ""
+	defaultMultusCNIVersion             = ""
+	defaultMultusConfigFile             = "auto"
+	defaultMultusGlobalNamespaces       = ""
+	defaultMultusKubeconfigPath         = "/etc/cni/net.d/multus.d/multus.kubeconfig"
+	defaultMultusLogFile                = ""
+	defaultMultusLogMaxSize             = 100 // megabytes
+	defaultMultusLogMaxAge              = 5   // days
+	defaultMultusLogMaxBackups          = 5
+	defaultMultusLogCompress            = true
+	defaultMultusLogLevel               = ""
+	defaultMultusLogToStdErr            = false
+	defaultMultusMasterCNIFile          = ""
+	defaultMultusNamespaceIsolation     = false
+	defaultMultusReadinessIndicatorFile = ""
+)
+
+const (
+	cniConfigDirVarName           = "cni-config-dir"
+	multusAdditionalBinDirVarName = "additional-bin-dir"
+	multusAutoconfigDirVarName    = "multus-autoconfig-dir"
+	multusCNIVersion              = "cni-version"
+	multusConfigFileVarName       = "multus-conf-file"
+	multusGlobalNamespaces        = "global-namespaces"
+	multusLogFile                 = "multus-log-file"
+	multusLogMaxSize              = "multus-log-max-size"
+	multusLogMaxAge               = "multus-log-max-age"
+	multusLogMaxBackups           = "multus-log-max-backups"
+	multusLogCompress             = "multus-log-compress"
+	multusLogLevel                = "multus-log-level"
+	multusLogToStdErr             = "multus-log-to-stderr"
+	multusKubeconfigPath          = "multus-kubeconfig-file-host"
+	multusMasterCNIFileVarName    = "multus-master-cni-file"
+	multusNamespaceIsolation      = "namespace-isolation"
+	multusReadinessIndicatorFile  = "readiness-indicator-file"
+)
+
+func main() {
+	versionOpt := false
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	multusConfigFile := flag.String(multusConfigFileVarName, defaultMultusConfigFile, "The multus configuration file to use. By default, a new configuration is generated.")
+	multusMasterCni := flag.String(multusMasterCNIFileVarName, defaultMultusMasterCNIFile, "The relative name of the configuration file of the cluster primary CNI.")
+	multusAutoconfigDir := flag.String(multusAutoconfigDirVarName, *cniConfigDir, "The directory path for the generated multus configuration.")
+	namespaceIsolation := flag.Bool(multusNamespaceIsolation, defaultMultusNamespaceIsolation, "If the network resources are only available within their defined namespaces.")
+	globalNamespaces := flag.String(multusGlobalNamespaces, defaultMultusGlobalNamespaces, "Comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled.")
+	logToStdErr := flag.Bool(multusLogToStdErr, defaultMultusLogToStdErr, "If the multus logs are also to be echoed to stderr.")
+	logLevel := flag.String(multusLogLevel, defaultMultusLogLevel, "One of: debug/verbose/error/panic. Used only with --multus-conf-file=auto.")
+	logFile := flag.String(multusLogFile, defaultMultusLogFile, "Path where to multus will log. Used only with --multus-conf-file=auto.")
+	logMaxSize := flag.Int(multusLogMaxSize, defaultMultusLogMaxSize, "the maximum size in megabytes of the log file before it gets rotated")
+	logMaxAge := flag.Int(multusLogMaxAge, defaultMultusLogMaxAge, "the maximum number of days to retain old log files in their filename")
+	logMaxBackups := flag.Int(multusLogMaxBackups, defaultMultusLogMaxBackups, "the maximum number of old log files to retain")
+	logCompress := flag.Bool(multusLogCompress, defaultMultusLogCompress, "compress determines if the rotated log files should be compressed using gzip")
+	cniVersion := flag.String(multusCNIVersion, defaultMultusCNIVersion, "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
+	additionalBinDir := flag.String(multusAdditionalBinDirVarName, defaultMultusAdditionalBinDir, "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
+	readinessIndicator := flag.String(multusReadinessIndicatorFile, defaultMultusReadinessIndicatorFile, "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
+	multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
+	overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	if *logToStdErr {
+		logging.SetLogStderr(*logToStdErr)
+	}
+	if *logFile != defaultMultusLogFile {
+		logging.SetLogFile(*logFile)
+	}
+	if *logLevel != defaultMultusLogLevel {
+		logging.SetLogLevel(*logLevel)
+	}
+
+	if *multusConfigFile == defaultMultusConfigFile {
+		if *cniVersion == defaultMultusCNIVersion {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		var configurationOptions []config.Option
+		if *namespaceIsolation {
+			configurationOptions = append(
+				configurationOptions, config.WithNamespaceIsolation())
+		}
+
+		if *globalNamespaces != defaultMultusGlobalNamespaces {
+			configurationOptions = append(
+				configurationOptions, config.WithGlobalNamespaces(*globalNamespaces))
+		}
+
+		if *logToStdErr != defaultMultusLogToStdErr {
+			configurationOptions = append(
+				configurationOptions, config.WithLogToStdErr())
+		}
+
+		if *logLevel != defaultMultusLogLevel {
+			configurationOptions = append(
+				configurationOptions, config.WithLogLevel(*logLevel))
+		}
+
+		if *logFile != defaultMultusLogFile {
+			configurationOptions = append(
+				configurationOptions, config.WithLogFile(*logFile))
+		}
+
+		if *additionalBinDir != defaultMultusAdditionalBinDir {
+			configurationOptions = append(
+				configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
+		}
+
+		if *readinessIndicator != defaultMultusReadinessIndicatorFile {
+			configurationOptions = append(
+				configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
+		}
+
+		// logOptions
+
+		var logOptionFuncs []config.LogOptionFunc
+		if *logMaxAge != defaultMultusLogMaxAge {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxAge(logMaxAge))
+		}
+		if *logMaxSize != defaultMultusLogMaxSize {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxSize(logMaxSize))
+		}
+		if *logMaxBackups != defaultMultusLogMaxBackups {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxBackups(logMaxBackups))
+		}
+		if *logCompress != defaultMultusLogCompress {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogCompress(logCompress))
+		}
+
+		if len(logOptionFuncs) > 0 {
+			logOptions := &config.LogOptions{}
+			config.MutateLogOptions(logOptions, logOptionFuncs...)
+			configurationOptions = append(configurationOptions, config.WithLogOptions(logOptions))
+		}
+
+		multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
+		if err != nil {
+			_ = logging.Errorf("Failed to create multus config: %v", err)
+			os.Exit(3)
+		}
+
+		var configManager *config.Manager
+		if *multusMasterCni == "" {
+			configManager, err = config.NewManager(*multusConfig, *multusAutoconfigDir)
+		} else {
+			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
+				*multusConfig, *multusAutoconfigDir, *multusMasterCni)
+		}
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+
+		if *overrideNetworkName {
+			if err := configManager.OverrideNetworkName(); err != nil {
+				_ = logging.Errorf("could not override the network name: %v", err)
+			}
+		}
+
+		generatedMultusConfig, err := configManager.GenerateConfig()
+		if err != nil {
+			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
+		}
+		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
+			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+		}
+
+		configWatcherDoneChannel := make(chan struct{})
+		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
+			defer func() {
+				stopChannel <- struct{}{}
+			}()
+			if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
+				_ = logging.Errorf("error watching file: %v", err)
+			}
+		}(make(chan struct{}), configWatcherDoneChannel)
+
+		<-configWatcherDoneChannel
+	} else {
+		if err := copyUserProvidedConfig(*multusConfigFile, *cniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", *multusConfigFile, err)
+		}
+	}
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	srcFile, err := os.Open(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(srcFile, dstFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/install_multus/main.go

@@ -1,67 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is a install tool for multus plugins
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/pflag"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
-)
-
-func main() {
-	typeFlag := pflag.StringP("type", "t", "", "specify installer type (thick/thin)")
-	destDir := pflag.StringP("dest-dir", "d", "/host/opt/cni/bin", "destination directory")
-	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
-
-	pflag.Parse()
-	if *helpFlag {
-		pflag.PrintDefaults()
-		os.Exit(1)
-	}
-
-	multusFileName := ""
-	switch *typeFlag {
-	case "thick":
-		multusFileName = "multus-shim"
-	case "thin":
-		multusFileName = "multus"
-	default:
-		fmt.Fprintf(os.Stderr, "--type is missing or --type has invalid value\n")
-		os.Exit(1)
-	}
-
-	err := cmdutils.CopyFileAtomic(fmt.Sprintf("/usr/src/multus-cni/bin/%s", multusFileName), *destDir, fmt.Sprintf("%s.temp", multusFileName), multusFileName)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
-		os.Exit(1)
-	}
-
-	fmt.Printf("multus %s copy succeeded!\n", multusFileName)
-
-	// Copy the passthru CNI
-	passthruPath := "/usr/src/multus-cni/bin/passthru"
-	err = cmdutils.CopyFileAtomic(passthruPath, *destDir, fmt.Sprintf("%s.temp", "passthru"), "passthru")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
-		os.Exit(1)
-	}
-
-	fmt.Printf("passthru cni %s copy succeeded!\n", passthruPath)
-
-}

cmd/kubeconfig_generator/main.go

@@ -1,145 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary submit CSR for kube controll access for multus thin plugin
-// and generate Kubeconfig
-package main
-
-import (
-	"encoding/base64"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"text/template"
-	"time"
-
-	"github.com/spf13/pflag"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
-
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog/v2"
-)
-
-var kubeConfigTemplate = `apiVersion: v1
-clusters:
-  - cluster:
-      certificate-authority-data: {{.CADATA}}
-      server: {{.K8S_APISERVER}}
-    name: default-cluster
-contexts:
-  - context:
-      cluster: default-cluster
-      namespace: default
-      user: default-auth
-    name: default-context
-current-context: default-context
-kind: Config
-preferences: {}
-users:
-  - name: default-auth
-    user:
-      client-certificate: {{.CERTDIR}}/multus-client-current.pem
-      client-key: {{.CERTDIR}}/multus-client-current.pem
-`
-
-func main() {
-	certDir := pflag.StringP("certdir", "", "/tmp", "specify cert directory")
-	bootstrapConfig := pflag.StringP("bootstrap-config", "", "/tmp/kubeconfig", "specify bootstrap kubernetes config")
-	kubeconfigPathRaw := pflag.StringP("kubeconfig", "", "/run/multus/kubeconfig", "specify output kubeconfig path")
-	certDurationString := pflag.StringP("cert-duration", "", "10m", "specify certificate duration")
-	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
-
-	kubeconfigPath, err := filepath.Abs(*kubeconfigPathRaw)
-	if err != nil {
-		klog.Fatalf("illegal path %s in kubeconfigPath %s: %v", kubeconfigPath, *kubeconfigPathRaw, err)
-	}
-
-	pflag.Parse()
-	if *helpFlag {
-		pflag.PrintDefaults()
-		os.Exit(1)
-	}
-
-	// check variables
-	if _, err := os.Stat(*bootstrapConfig); err != nil {
-		klog.Fatalf("failed to read bootstrap config %q", *bootstrapConfig)
-	}
-	st, err := os.Stat(*certDir)
-	if err != nil {
-		klog.Fatalf("failed to find cert directory %q", *certDir)
-	}
-	if !st.IsDir() {
-		klog.Fatalf("cert directory %q is not directory", *certDir)
-	}
-	certDuration, err := time.ParseDuration(*certDurationString)
-	if err != nil {
-		klog.Fatalf("failed to parse duration %q: %v", *certDurationString, err)
-	}
-
-	nodeName := os.Getenv("MULTUS_NODE_NAME")
-	if nodeName == "" {
-		klog.Fatalf("cannot identify node name from MULTUS_NODE_NAME env variables")
-	}
-
-	// retrieve API server from bootstrapConfig()
-	config, err := clientcmd.BuildConfigFromFlags("", *bootstrapConfig)
-	if err != nil {
-		klog.Fatalf("cannot get in-cluster config: %v", err)
-	}
-	apiServer := fmt.Sprintf("%s%s", config.Host, config.APIPath)
-	caData := base64.StdEncoding.EncodeToString(config.CAData)
-
-	// run certManager to create certification
-	if _, err = k8sclient.PerNodeK8sClient(nodeName, *bootstrapConfig, certDuration, *certDir); err != nil {
-		klog.Fatalf("failed to start cert manager: %v", err)
-	}
-
-	fp, err := os.OpenFile(kubeconfigPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		klog.Fatalf("cannot create kubeconfig file %q: %v", kubeconfigPath, err)
-	}
-
-	// render kubeconfig
-	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
-	if err != nil {
-		klog.Fatalf("template parse error: %v", err)
-	}
-	templateData := map[string]string{
-		"CADATA":        caData,
-		"CERTDIR":       *certDir,
-		"K8S_APISERVER": apiServer,
-	}
-	// genearate kubeconfig from template
-	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
-		klog.Fatalf("cannot create kubeconfig: %v", err)
-	}
-	if err = fp.Close(); err != nil {
-		klog.Fatalf("cannot save kubeconfig: %v", err)
-	}
-
-	klog.Infof("kubeconfig %q is saved", kubeconfigPath)
-
-	// wait for signal
-	sigterm := make(chan os.Signal, 1)
-	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
-	<-sigterm
-	klog.Infof("signal received. remove kubeconfig %q and quit.", kubeconfigPath)
-	err = os.Remove(kubeconfigPath)
-	if err != nil {
-		klog.Errorf("failed to remove kubeconfig %q: %v", kubeconfigPath, err)
-	}
-}

cmd/main.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept referred from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+func main() {
+
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			result, err := multus.CmdAdd(args, nil, nil)
+			if err != nil {
+				return err
+			}
+			return result.Print()
+		},
+		func(args *skel.CmdArgs) error {
+			return multus.CmdCheck(args, nil, nil)
+		},
+		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

cmd/multus-daemon/main.go

@@ -1,220 +0,0 @@
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary works as a server that receives requests from multus-shim
-// CNI plugin and creates network interface for kubernets pods.
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/signal"
-	"os/user"
-	"path/filepath"
-	"sync"
-	"syscall"
-
-	utilwait "k8s.io/apimachinery/pkg/util/wait"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
-	srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
-
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-)
-
-func main() {
-	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-	// keep in command line option
-	version := flag.Bool("version", false, "Show version")
-
-	configFilePath := flag.String("config", srv.DefaultMultusDaemonConfigFile, "Specify the path to the multus-daemon configuration")
-
-	flag.Parse()
-
-	if *version {
-		fmt.Printf("multus-daemon: %s\n", multus.PrintVersionString())
-		os.Exit(4)
-	}
-
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-
-	daemonConf, err := cniServerConfig(*configFilePath)
-	if err != nil {
-		logging.Panicf("startMultusDaemon failed to load the CNI server configuration: %v", err)
-		os.Exit(1)
-	}
-
-	multusConf, err := config.ParseMultusConfig(*configFilePath)
-	if err != nil {
-		logging.Panicf("startMultusDaemon failed to load the multus configuration: %v", err)
-		os.Exit(1)
-	}
-
-	logging.Verbosef("multus-daemon started")
-
-	if multusConf.ReadinessIndicatorFile != "" {
-		// Check readinessindicator file before daemon launch
-		logging.Verbosef("Readiness Indicator file check")
-		if err := types.GetReadinessIndicatorFile(multusConf.ReadinessIndicatorFile); err != nil {
-			_ = logging.Errorf("have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", multusConf.ReadinessIndicatorFile, err)
-			os.Exit(1)
-		}
-		logging.Verbosef("Readiness Indicator file check done!")
-	}
-
-	var configManager *config.Manager
-	var ignoreReadinessIndicator bool
-	if multusConf.MultusConfigFile == "auto" {
-		if multusConf.CNIVersion == "" {
-			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
-		}
-
-		// Generate multus CNI config from current CNI config
-		configManager, err = config.NewManager(*multusConf)
-		if err != nil {
-			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
-			os.Exit(2)
-		}
-		// ConfigManager watches the readiness indicator file (if configured)
-		// and exits the daemon when that is removed. The CNIServer does
-		// not need to re-do that check every CNI operation
-		ignoreReadinessIndicator = true
-	} else {
-		if err := copyUserProvidedConfig(multusConf.MultusConfigFile, multusConf.CniConfigDir); err != nil {
-			logging.Errorf("failed to copy the user provided configuration %s: %v", multusConf.MultusConfigFile, err)
-		}
-	}
-
-	if err := startMultusDaemon(ctx, daemonConf, ignoreReadinessIndicator); err != nil {
-		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
-		os.Exit(3)
-	}
-
-	// Wait until daemon ready
-	logging.Verbosef("API readiness check")
-	if api.WaitUntilAPIReady(daemonConf.SocketDir) != nil {
-		logging.Panicf("failed to ready multus-daemon socket: %v", err)
-		os.Exit(1)
-	}
-	logging.Verbosef("API readiness check done!")
-
-	signalCh := make(chan os.Signal, 16)
-	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		for sig := range signalCh {
-			logging.Verbosef("caught %v, stopping...", sig)
-			cancel()
-		}
-	}()
-
-	var wg sync.WaitGroup
-	if configManager != nil {
-		if err := configManager.Start(ctx, &wg); err != nil {
-			_ = logging.Errorf("failed to start config manager: %v", err)
-			os.Exit(3)
-		}
-	}
-
-	wg.Wait()
-	logging.Verbosef("multus daemon is exited")
-}
-
-func startMultusDaemon(ctx context.Context, daemonConfig *srv.ControllerNetConf, ignoreReadinessIndicator bool) error {
-	if user, err := user.Current(); err != nil || user.Uid != "0" {
-		return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
-	}
-
-	if err := srv.FilesystemPreRequirements(daemonConfig.SocketDir); err != nil {
-		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
-	}
-
-	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents, ignoreReadinessIndicator)
-	if err != nil {
-		return fmt.Errorf("failed to create the server: %v", err)
-	}
-
-	if daemonConfig.MetricsPort != nil {
-		go utilwait.UntilWithContext(ctx, func(_ context.Context) {
-			http.Handle("/metrics", promhttp.Handler())
-			logging.Debugf("metrics port: %d", *daemonConfig.MetricsPort)
-			logging.Debugf("metrics: %s", http.ListenAndServe(fmt.Sprintf(":%d", *daemonConfig.MetricsPort), nil))
-		}, 0)
-	}
-
-	l, err := srv.GetListener(api.SocketPath(daemonConfig.SocketDir))
-	if err != nil {
-		return fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(daemonConfig.SocketDir), err)
-	}
-
-	server.Start(ctx, l)
-
-	go func() {
-		<-ctx.Done()
-		server.Shutdown(context.Background())
-	}()
-
-	return nil
-}
-
-func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
-	path, err := filepath.Abs(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("illegal path %s in server config path %s: %w", path, configFilePath, err)
-	}
-
-	configFileContents, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	return srv.LoadDaemonNetConf(configFileContents)
-}
-
-func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
-	path, err := filepath.Abs(multusConfigPath)
-	if err != nil {
-		return fmt.Errorf("illegal path %s in multusConfigPath %s: %w", path, multusConfigPath, err)
-	}
-
-	srcFile, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("failed to open (READ only) file %s: %w", path, err)
-	}
-
-	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
-	dstFile, err := os.Create(dstFileName)
-	if err != nil {
-		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
-	}
-	nBytes, err := io.Copy(dstFile, srcFile)
-	if err != nil {
-		return fmt.Errorf("error copying file: %w", err)
-	}
-	srcFileInfo, err := srcFile.Stat()
-	if err != nil {
-		return fmt.Errorf("failed to stat the file: %w", err)
-	} else if nBytes != srcFileInfo.Size() {
-		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
-	}
-	return nil
-}

cmd/multus-shim/main.go

@@ -1,66 +0,0 @@
-// Copyright (c) 2022 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is a "Multi-plugin".The delegate concept referred from CNI project
-// It reads other plugin netconf, and then invoke them, e.g.
-// flannel or sriov plugin.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-
-	"github.com/containernetworking/cni/pkg/skel"
-	cniversion "github.com/containernetworking/cni/pkg/version"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
-)
-
-func main() {
-	// Init command line flags to clear vendored packages' one, especially in init()
-	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-	// add version flag
-	versionOpt := false
-	flag.BoolVar(&versionOpt, "version", false, "Show application version")
-	flag.BoolVar(&versionOpt, "v", false, "Show application version")
-
-	flag.Parse()
-	if versionOpt {
-		fmt.Printf("multus-shim: %s\n", multus.PrintVersionString())
-		return
-	}
-
-	skel.PluginMainFuncs(
-		skel.CNIFuncs{
-			Add: func(args *skel.CmdArgs) error {
-				return api.CmdAdd(args)
-			},
-			Check: func(args *skel.CmdArgs) error {
-				return api.CmdCheck(args)
-			},
-			Del: func(args *skel.CmdArgs) error {
-				return api.CmdDel(args)
-			},
-			GC: func(args *skel.CmdArgs) error {
-				return api.CmdGC(args)
-			},
-			Status: func(args *skel.CmdArgs) error {
-				return api.CmdStatus(args)
-			},
-		},
-		cniversion.All, "meta-plugin that delegates to other CNI plugins")
-}

cmd/multus/main.go

@@ -1,69 +0,0 @@
-// Copyright (c) 2016 Intel Corporation
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is a "Multi-plugin".The delegate concept referred from CNI project
-// It reads other plugin netconf, and then invoke them, e.g.
-// flannel or sriov plugin.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-
-	"github.com/containernetworking/cni/pkg/skel"
-	cniversion "github.com/containernetworking/cni/pkg/version"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
-)
-
-func main() {
-
-	// Init command line flags to clear vendored packages' one, especially in init()
-	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-	// add version flag
-	versionOpt := false
-	flag.BoolVar(&versionOpt, "version", false, "Show application version")
-	flag.BoolVar(&versionOpt, "v", false, "Show application version")
-	flag.Parse()
-	if versionOpt {
-		fmt.Printf("multus: %s\n", multus.PrintVersionString())
-		return
-	}
-
-	skel.PluginMainFuncs(
-		skel.CNIFuncs{
-			Add: func(args *skel.CmdArgs) error {
-				result, err := multus.CmdAdd(args, nil, nil)
-				if err != nil {
-					return err
-				}
-				return result.Print()
-			},
-			Del: func(args *skel.CmdArgs) error {
-				return multus.CmdDel(args, nil, nil)
-			},
-			Check: func(args *skel.CmdArgs) error {
-				return multus.CmdCheck(args, nil, nil)
-			},
-			GC: func(args *skel.CmdArgs) error {
-				return multus.CmdGC(args, nil, nil)
-			},
-			Status: func(args *skel.CmdArgs) error {
-				return multus.CmdStatus(args, nil, nil)
-			},
-		},
-		cniversion.All, "meta-plugin that delegates to other CNI plugins")
-}

cmd/passthru-cni/main.go

@@ -1,58 +0,0 @@
-// Package: passthru-cni
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/containernetworking/cni/pkg/skel"
-	cniTypes "github.com/containernetworking/cni/pkg/types"
-	current "github.com/containernetworking/cni/pkg/types/100"
-	cniVersion "github.com/containernetworking/cni/pkg/version"
-)
-
-// NetConf is a CNI configuration structure
-type NetConf struct {
-	cniTypes.NetConf
-}
-
-func main() {
-	skel.PluginMain(
-		cmdAdd,
-		nil,
-		cmdDel,
-		cniVersion.PluginSupports("0.3.0", "0.3.1", "0.4.0", "1.0.0", "1.1.0"),
-		"Passthrough CNI Plugin v1.0",
-	)
-}
-
-func cmdAdd(args *skel.CmdArgs) error {
-	n, err := loadNetConf(args.StdinData)
-	if err != nil {
-		return fmt.Errorf("passthru cni: error parsing CNI configuration: %s", err)
-	}
-
-	// Create an empty but valid CNI result
-	result := &current.Result{
-		CNIVersion: n.CNIVersion,
-		Interfaces: []*current.Interface{},
-		IPs:        []*current.IPConfig{},
-		Routes:     []*cniTypes.Route{},
-		DNS:        cniTypes.DNS{},
-	}
-
-	return cniTypes.PrintResult(result, n.CNIVersion)
-}
-
-func cmdDel(_ *skel.CmdArgs) error {
-	// Nothing to do for DEL command, just return nil
-	return nil
-}
-
-func loadNetConf(bytes []byte) (*NetConf, error) {
-	n := &NetConf{}
-	if err := json.Unmarshal(bytes, n); err != nil {
-		return nil, fmt.Errorf("passthru cni: failed to load netconf: %s", err)
-	}
-	return n, nil
-}

cmd/thin_entrypoint/main.go

@@ -1,683 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is a entrypoint for thin (stand-alone) images.
-package main
-
-import (
-	"bytes"
-	"crypto/sha256"
-	b64 "encoding/base64"
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"text/template"
-	"time"
-
-	"github.com/containernetworking/cni/libcni"
-	"github.com/spf13/pflag"
-
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
-)
-
-// Options stores command line options
-type Options struct {
-	CNIBinDir                string
-	CNIConfDir               string
-	CNIVersion               string
-	MultusConfFile           string
-	MultusBinFile            string // may be hidden or remove?
-	MultusCNIConfDir         string
-	SkipMultusBinaryCopy     bool
-	MultusKubeConfigFileHost string
-	MultusMasterCNIFileName  string
-	NamespaceIsolation       bool
-	GlobalNamespaces         string
-	MultusAutoconfigDir      string
-	MultusLogToStderr        bool
-	MultusLogLevel           string
-	MultusLogFile            string
-	OverrideNetworkName      bool
-	CleanupConfigOnExit      bool
-	RenameConfFile           bool
-	ReadinessIndicatorFile   string
-	AdditionalBinDir         string
-	ForceCNIVersion          bool
-	SkipTLSVerify            bool
-	SkipMultusConfWatch      bool
-}
-
-const (
-	serviceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-	serviceAccountCAFile    = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-)
-
-func (o *Options) addFlags() {
-	pflag.ErrHelp = nil // suppress error message for help
-	fs := pflag.CommandLine
-	fs.StringVar(&o.CNIBinDir, "cni-bin-dir", "/host/opt/cni/bin", "CNI binary directory")
-	fs.StringVar(&o.CNIConfDir, "cni-conf-dir", "/host/etc/cni/net.d", "CNI config directory")
-	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
-	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
-	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
-	fs.StringVar(&o.MultusCNIConfDir, "multus-cni-conf-dir", "/host/etc/cni/multus/net.d", "multus specific CNI config directory")
-	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")
-
-	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
-	fs.StringVar(&o.MultusMasterCNIFileName, "multus-master-cni-file-name", "", "master CNI file in multus-autoconfig-dir")
-	fs.BoolVar(&o.NamespaceIsolation, "namespace-isolation", false, "namespace isolation")
-	fs.StringVar(&o.GlobalNamespaces, "global-namespaces", "", "global namespaces, comma separated (used only with --namespace-isolation=true)")
-	fs.StringVar(&o.MultusAutoconfigDir, "multus-autoconfig-dir", "/host/etc/cni/net.d", "multus autoconfig dir (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.MultusLogToStderr, "multus-log-to-stderr", true, "log to stderr")
-	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
-	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
-	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
-	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
-	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
-	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.SkipTLSVerify, "skip-tls-verify", false, "skip TLS verify")
-	fs.BoolVar(&o.ForceCNIVersion, "force-cni-version", false, "force cni version to '--cni-version' (only for e2e-kind testing)")
-	fs.MarkHidden("force-cni-version")
-	fs.MarkHidden("skip-tls-verify")
-}
-
-func (o *Options) verifyFileExists() error {
-	// CNIConfDir
-	if _, err := os.Stat(o.CNIConfDir); err != nil {
-		return fmt.Errorf("cni-conf-dir is not found: %v", err)
-	}
-
-	// CNIBinDir
-	if _, err := os.Stat(o.CNIBinDir); err != nil {
-		return fmt.Errorf("cni-bin-dir is not found: %v", err)
-	}
-
-	// MultusBinFile
-	if _, err := os.Stat(o.MultusBinFile); err != nil {
-		return fmt.Errorf("multus-bin-file is not found: %v", err)
-	}
-
-	if o.MultusConfFile != "auto" {
-		// MultusConfFile
-		if _, err := os.Stat(o.MultusConfFile); err != nil {
-			return fmt.Errorf("multus-conf-file is not found: %v", err)
-		}
-	}
-	return nil
-}
-
-const kubeConfigTemplate = `# Kubeconfig file for Multus CNI plugin.
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: {{ .KubeConfigHost }}
-    {{ .KubeServerTLS }}
-users:
-- name: multus
-  user:
-    token: "{{ .KubeServiceAccountToken }}"
-contexts:
-- name: multus-context
-  context:
-    cluster: local
-    user: multus
-current-context: multus-context
-`
-
-func getFileAndHash(filepath string) ([]byte, []byte, error) {
-	if _, err := os.Stat(filepath); err != nil {
-		return nil, nil, fmt.Errorf("file %s not found: %v", filepath, err)
-	}
-	content, err := os.ReadFile(filepath)
-	if err != nil {
-		return nil, nil, fmt.Errorf("cannot read %s file: %v", filepath, err)
-	}
-
-	hash := sha256.New()
-	hash.Write(content)
-	return content, hash.Sum(nil), nil
-}
-
-func (o *Options) createKubeConfig(prevCAHash, prevSATokenHash []byte) ([]byte, []byte, error) {
-	caFileByte, caHash, err := getFileAndHash(serviceAccountCAFile)
-	if err != nil {
-		return nil, nil, err
-	}
-	saTokenByte, saTokenHash, err := getFileAndHash(serviceAccountTokenFile)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	caUnchanged := prevCAHash != nil && bytes.Equal(prevCAHash, caHash)
-	saUnchanged := prevSATokenHash != nil && bytes.Equal(prevSATokenHash, saTokenHash)
-
-	if o.SkipTLSVerify {
-		if saUnchanged {
-			return caHash, saTokenHash, nil
-		}
-	} else {
-		if caUnchanged && saUnchanged {
-			return caHash, saTokenHash, nil
-		}
-	}
-
-	if prevSATokenHash != nil {
-		// don't log "recreating" on first function execution
-		fmt.Printf("CA (%v) or SA token (%v) changed - recreating kubeconfig\n", !caUnchanged, !saUnchanged)
-	}
-
-	// create multus.d directory
-	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
-		return nil, nil, fmt.Errorf("cannot create multus.d directory: %v", err)
-	}
-
-	// create multus cni conf directory
-	if err := os.MkdirAll(o.MultusCNIConfDir, 0755); err != nil {
-		return nil, nil, fmt.Errorf("cannot create multus-cni-conf-dir(%s) directory: %v", o.MultusCNIConfDir, err)
-	}
-
-	// get Kubernetes service protocol/host/port
-	kubeProtocol := os.Getenv("KUBERNETES_SERVICE_PROTOCOL")
-	if kubeProtocol == "" {
-		kubeProtocol = "https"
-	}
-	kubeHost := os.Getenv("KUBERNETES_SERVICE_HOST")
-	kubePort := os.Getenv("KUBERNETES_SERVICE_PORT")
-
-	// check tlsConfig
-	tlsConfig := ""
-	if o.SkipTLSVerify {
-		tlsConfig = "insecure-skip-tls-verify: true"
-	} else {
-		// create tlsConfig by service account CA file
-		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
-		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
-	}
-
-	// create kubeconfig by template and replace it by atomic
-	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
-	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
-	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return nil, nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
-	}
-
-	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
-	if err != nil {
-		return nil, nil, fmt.Errorf("template parse error: %v", err)
-	}
-	templateData := map[string]string{
-		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
-		"KubeServerTLS":           tlsConfig,
-		"KubeServiceAccountToken": string(saTokenByte),
-	}
-
-	// generate kubeconfig from template
-	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
-		return nil, nil, fmt.Errorf("cannot create kubeconfig: %v", err)
-	}
-
-	if err := fp.Sync(); err != nil {
-		os.Remove(fp.Name())
-		return nil, nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
-	}
-	if err := fp.Close(); err != nil {
-		os.Remove(fp.Name())
-		return nil, nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
-	}
-
-	// replace file with tempfile
-	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
-		return nil, nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
-	}
-
-	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
-	return caHash, saTokenHash, nil
-}
-
-const multusConflistTemplate = `{
-    "cniVersion": "{{ .CNIVersion }}",
-    "name": "{{ .MasterPluginNetworkName }}",
-    "plugins": [ {
-        "type": "multus",{{
-            .NestedCapabilities
-        }}{{
-            .NamespaceIsolationConfig
-        }}{{
-            .GlobalNamespacesConfig
-        }}{{
-            .LogToStderrConfig
-        }}{{
-            .LogLevelConfig
-        }}{{
-            .LogFileConfig
-        }}{{
-            .AdditionalBinDirConfig
-        }}{{
-            .MultusCNIConfDirConfig
-        }}{{
-            .ReadinessIndicatorFileConfig
-        }}
-        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
-        "delegates": [
-            {{ .MasterPluginJSON }}
-        ]
-    }]
-}
-`
-
-const multusConfTemplate = `{
-        "cniVersion": "{{ .CNIVersion }}",
-        "name": "{{ .MasterPluginNetworkName }}",
-        "type": "multus",{{
-            .NestedCapabilities
-        }}{{
-            .NamespaceIsolationConfig
-        }}{{
-            .GlobalNamespacesConfig
-        }}{{
-            .LogToStderrConfig
-        }}{{
-            .LogLevelConfig
-        }}{{
-            .LogFileConfig
-        }}{{
-            .AdditionalBinDirConfig
-        }}{{
-            .MultusCNIConfDirConfig
-        }}{{
-            .ReadinessIndicatorFileConfig
-        }}
-        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
-        "delegates": [
-                {{ .MasterPluginJSON }}
-        ]
-}
-`
-
-func (o *Options) getMasterConfigPath() (string, error) {
-	// Master config file is specified
-	if o.MultusMasterCNIFileName != "" {
-		return filepath.Join(o.MultusAutoconfigDir, o.MultusMasterCNIFileName), nil
-	}
-
-	// Pick the alphabetically first config file from MultusAutoconfigDir
-	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
-	if err != nil {
-		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
-	}
-
-	for _, filename := range files {
-		if !strings.HasPrefix(filepath.Base(filename), "00-multus.conf") {
-			return filename, nil
-		}
-	}
-
-	// No config file found
-	return "", fmt.Errorf("cannot find valid master CNI config in %q", o.MultusAutoconfigDir)
-}
-
-func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, []byte, error) {
-	masterConfigPath, err := o.getMasterConfigPath()
-	if err != nil {
-		return "", nil, err
-	}
-
-	masterConfigBytes, masterConfigFileHash, err := getFileAndHash(masterConfigPath)
-	if err != nil {
-		return "", nil, err
-	}
-
-	if prevMasterConfigFileHash != nil && bytes.Equal(prevMasterConfigFileHash, masterConfigFileHash) {
-		return masterConfigPath, masterConfigFileHash, nil
-	}
-
-	if prevMasterConfigFileHash != nil {
-		// don't log "recreating" on first function execution
-		fmt.Printf("master config changed - recreating multus config\n")
-	}
-
-	masterConfig := map[string]interface{}{}
-	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
-		return "", nil, fmt.Errorf("cannot read master CNI config json: %v", err)
-	}
-
-	// check CNIVersion
-	masterCNIVersionElem, ok := masterConfig["cniVersion"]
-	if !ok {
-		return "", nil, fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
-	}
-
-	if o.ForceCNIVersion {
-		masterConfig["cniVersion"] = o.CNIVersion
-		fmt.Printf("force CNI version to %q\n", o.CNIVersion)
-	} else {
-		masterCNIVersion := masterCNIVersionElem.(string)
-		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
-			return "", nil, fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
-		}
-		o.CNIVersion = masterCNIVersion
-	}
-	cniVersionConfig := o.CNIVersion
-
-	// check OverrideNetworkName (if true, get master plugin name, otherwise 'multus-cni-network'
-	masterPluginNetworkName := "multus-cni-network"
-	if o.OverrideNetworkName {
-		masterPluginNetworkElem, ok := masterConfig["name"]
-		if !ok {
-			return "", nil, fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
-		}
-
-		masterPluginNetworkName = masterPluginNetworkElem.(string)
-		fmt.Printf("master plugin name is overrided to %q\n", masterPluginNetworkName)
-	}
-
-	// check capabilities (from master conf, top and 'plugins')
-	masterCapabilities := map[string]bool{}
-	_, isMasterConfList := masterConfig["plugins"]
-
-	if isMasterConfList {
-		masterPluginsElem, ok := masterConfig["plugins"]
-		if !ok {
-			return "", nil, fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
-		}
-		masterPlugins := masterPluginsElem.([]interface{})
-		for _, v := range masterPlugins {
-			pluginFields := v.(map[string]interface{})
-			capabilitiesElem, ok := pluginFields["capabilities"]
-			if ok {
-				capabilities := capabilitiesElem.(map[string]interface{})
-				for k, v := range capabilities {
-					masterCapabilities[k] = v.(bool)
-				}
-			}
-		}
-		fmt.Printf("master capabilities is get from conflist\n")
-	} else {
-		masterCapabilitiesElem, ok := masterConfig["capabilities"]
-		if ok {
-			for k, v := range masterCapabilitiesElem.(map[string]interface{}) {
-				masterCapabilities[k] = v.(bool)
-			}
-		}
-		fmt.Printf("master capabilities is get from conffile\n")
-	}
-	nestedCapabilitiesConf := ""
-	if len(masterCapabilities) != 0 {
-		capabilitiesByte, err := json.Marshal(masterCapabilities)
-		if err != nil {
-			return "", nil, fmt.Errorf("cannot get capabilities map: %v", err)
-		}
-		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
-	}
-
-	// check NamespaceIsolation
-	namespaceIsolationConfig := ""
-	if o.NamespaceIsolation {
-		namespaceIsolationConfig = "\n        \"namespaceIsolation\": true,"
-	}
-
-	// check GlobalNamespaces
-	globalNamespaceConfig := ""
-	if o.GlobalNamespaces != "" {
-		globalNamespaceConfig = fmt.Sprintf("\n        \"globalNamespaces\": %q,", o.GlobalNamespaces)
-	}
-
-	// check MultusLogToStderr
-	logToStderrConfig := ""
-	if !o.MultusLogToStderr {
-		logToStderrConfig = "\n        \"logToStderr\": false,"
-	}
-
-	// check MultusLogLevel (debug/error/panic/verbose) and reject others
-	logLevelConfig := ""
-	logLevelStr := strings.ToLower(o.MultusLogLevel)
-	switch logLevelStr {
-	case "debug", "error", "panic", "verbose":
-		logLevelConfig = fmt.Sprintf("\n        \"logLevel\": %q,", logLevelStr)
-	case "":
-		// no logLevel config, skipped
-	default:
-		return "", nil, fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
-	}
-
-	// check MultusLogFile
-	logFileConfig := ""
-	if o.MultusLogFile != "" {
-		logFileConfig = fmt.Sprintf("\n        \"logFile\": %q,", o.MultusLogFile)
-	}
-
-	// check AdditionalBinDir
-	additionalBinDirConfig := ""
-	if o.AdditionalBinDir != "" {
-		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
-	}
-
-	// check MultusCNIConfDir
-	multusCNIConfDirConfig := ""
-	if o.MultusCNIConfDir != "" {
-		multusCNIConfDirConfig = fmt.Sprintf("\n        \"cniConf\": %q,", o.MultusCNIConfDir)
-	}
-
-	// check ReadinessIndicatorFile
-	readinessIndicatorFileConfig := ""
-	if o.ReadinessIndicatorFile != "" {
-		readinessIndicatorFileConfig = fmt.Sprintf("\n        \"readinessindicatorfile\": %q,", o.ReadinessIndicatorFile)
-	}
-
-	// fill .MasterPluginJSON
-	masterPluginByte, err := json.Marshal(masterConfig)
-	if err != nil {
-		return "", nil, fmt.Errorf("cannot encode master CNI config: %v", err)
-	}
-
-	// generate multus config
-	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
-	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
-	if err != nil {
-		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
-	}
-
-	// use conflist template if cniVersionConfig == "1.0.0"
-	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
-	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
-	if err != nil {
-		return "", nil, fmt.Errorf("template parse error: %v", err)
-	}
-
-	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
-		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
-		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
-		if err != nil {
-			return "", nil, fmt.Errorf("template parse error: %v", err)
-		}
-	}
-
-	templateData := map[string]string{
-		"CNIVersion":                   cniVersionConfig,
-		"MasterPluginNetworkName":      masterPluginNetworkName,
-		"NestedCapabilities":           nestedCapabilitiesConf,
-		"NamespaceIsolationConfig":     namespaceIsolationConfig,
-		"GlobalNamespacesConfig":       globalNamespaceConfig,
-		"LogToStderrConfig":            logToStderrConfig,
-		"LogLevelConfig":               logLevelConfig,
-		"LogFileConfig":                logFileConfig,
-		"AdditionalBinDirConfig":       additionalBinDirConfig,
-		"MultusCNIConfDirConfig":       multusCNIConfDirConfig,
-		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
-		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
-		"MasterPluginJSON":             string(masterPluginByte),
-	}
-	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
-		return "", nil, fmt.Errorf("cannot create multus cni config: %v", err)
-	}
-
-	if err := fp.Sync(); err != nil {
-		os.Remove(tempFileName)
-		return "", nil, fmt.Errorf("cannot flush multus cni config: %v", err)
-	}
-	if err := fp.Close(); err != nil {
-		os.Remove(tempFileName)
-		return "", nil, fmt.Errorf("cannot close multus cni config: %v", err)
-	}
-
-	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
-		return "", nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
-	}
-
-	if o.RenameConfFile {
-		//masterConfigPath
-		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
-		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
-			return "", nil, fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
-		}
-		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
-	}
-
-	return masterConfigPath, masterConfigFileHash, nil
-}
-
-func main() {
-	opt := Options{}
-	opt.addFlags()
-	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
-
-	pflag.Parse()
-	if *helpFlag {
-		pflag.PrintDefaults()
-		os.Exit(1)
-	}
-
-	err := opt.verifyFileExists()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		return
-	}
-
-	// copy multus binary
-	if !opt.SkipMultusBinaryCopy {
-		// Copy
-		if err = cmdutils.CopyFileAtomic(opt.MultusBinFile, opt.CNIBinDir, "_multus", "multus"); err != nil {
-			fmt.Fprintf(os.Stderr, "failed at multus copy: %v\n", err)
-			return
-		}
-	}
-
-	var masterConfigHash, caHash, saTokenHash []byte
-	var masterConfigFilePath string
-	// copy user specified multus conf to CNI conf directory
-	if opt.MultusConfFile != "auto" {
-		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
-			return
-		}
-		confFileName := filepath.Base(opt.MultusConfFile)
-		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
-		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
-			fmt.Fprintf(os.Stderr, "failed at copy multus conf file: %v\n", err)
-			return
-		}
-		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
-	} else { // auto generate multus config
-		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
-			return
-		}
-		fmt.Printf("kubeconfig file is created.\n")
-		masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(nil)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
-			return
-		}
-		fmt.Printf("multus config file is created.\n")
-	}
-
-	ctx := signals.SetupSignalHandler()
-
-	if opt.CleanupConfigOnExit {
-		defer cleanupMultusConf(&opt)
-	}
-
-	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
-	if watchChanges {
-		fmt.Printf("Entering watch loop...\n")
-		masterConfigExists := true
-
-	outer:
-		for range time.Tick(1 * time.Second) {
-			select {
-			case <-ctx.Done():
-				// signal received break from loop
-				break outer
-			default:
-				// Check kubeconfig and update if different (i.e. service account updated)
-				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
-				if err != nil {
-					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
-					return
-				}
-
-				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
-				_, err = os.Stat(masterConfigFilePath)
-
-				// if masterConfigFilePath is no longer exists
-				if os.IsNotExist(err) {
-					if masterConfigExists {
-						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
-					}
-					masterConfigExists = false
-					continue
-				}
-
-				if !masterConfigExists {
-					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
-					masterConfigExists = true
-				}
-
-				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
-				if err != nil {
-					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
-					return
-				}
-			}
-		}
-	} else {
-		// wait until signal received
-		<-ctx.Done()
-	}
-}
-
-func cleanupMultusConf(opt *Options) {
-	// try remove multus conf
-	if opt.MultusConfFile == "auto" {
-		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
-		_ = os.Remove(multusConfFilePath)
-
-		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
-		_ = os.Remove(multusConfFilePath)
-	} else {
-		confFileName := filepath.Base(opt.MultusConfFile)
-		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
-	}
-
-}

cmd/thin_entrypoint/main_test.go

@@ -1,544 +0,0 @@
-package main
-
-// disable dot-imports only for testing
-//revive:disable:dot-imports
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-	"testing"
-
-	. "github.com/onsi/ginkgo/v2" //nolint:golint
-	. "github.com/onsi/gomega"    //nolint:golint
-)
-
-// chrootTestHelper performs chroot syscall, returns func to get back to original root or error if occurred
-func chrootTestHelper(path string) (func() error, error) {
-	root, err := os.Open("/")
-	if err != nil {
-		return nil, err
-	}
-
-	if err := syscall.Chroot(path); err != nil {
-		root.Close()
-		return nil, err
-	}
-
-	return func() error {
-		defer root.Close()
-		if err := root.Chdir(); err != nil {
-			return err
-		}
-		return syscall.Chroot(".")
-	}, nil
-}
-
-func TestThinEntrypoint(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "thin_entrypoint")
-}
-
-var _ = Describe("thin entrypoint testing", func() {
-	It("always pass just example", func() {
-		a := 10
-		Expect(a).To(Equal(10))
-	})
-
-	It("Run verifyFileExists() with expected environment, autoconfig", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
-		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
-		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
-		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
-
-		// CNIConfDir
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// CNIBinDir
-		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
-
-		// MultusBinFile
-		Expect(os.WriteFile(multusBinFile, nil, 0744)).To(Succeed())
-
-		// MultusConfFile
-		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
-
-		err = (&Options{
-			CNIConfDir:     cniConfDir,
-			CNIBinDir:      cniBinDir,
-			MultusBinFile:  multusBinFile,
-			MultusConfFile: multusConfFile,
-		}).verifyFileExists()
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run verifyFileExists() with invalid environmentMultusConfFile", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
-		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
-		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
-		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
-
-		// CNIConfDir
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// CNIBinDir
-		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
-
-		// MultusConfFile
-		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
-
-		err = (&Options{
-			CNIConfDir:     cniConfDir,
-			CNIBinDir:      cniBinDir,
-			MultusBinFile:  multusBinFile,
-			MultusConfFile: multusConfFile,
-		}).verifyFileExists()
-		Expect(err).To(HaveOccurred())
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), default, conf", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "0.3.1",
-			"name": "test1",
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult := `{
-        "cniVersion": "0.3.1",
-        "name": "multus-cni-network",
-        "type": "multus",
-        "logToStderr": false,
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
-        ]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), capabilities, conf", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "0.3.1",
-			"name": "test1",
-			"capabilities": { "bandwidth": true },
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult := `{
-        "cniVersion": "0.3.1",
-        "name": "multus-cni-network",
-        "type": "multus",
-        "capabilities": {"bandwidth":true},
-        "logToStderr": false,
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-                {"capabilities":{"bandwidth":true},"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
-        ]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), with options, conf", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "0.3.1",
-			"name": "test1",
-			"type": "cnitesttype"
-		}`
-		err = os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)
-		Expect(err).NotTo(HaveOccurred())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-			NamespaceIsolation:       true,
-			GlobalNamespaces:         "foobar,barfoo",
-			MultusLogToStderr:        true,
-			MultusLogLevel:           "DEBUG",
-			MultusLogFile:            "/tmp/foobar.log",
-			AdditionalBinDir:         "/tmp/add_bin_dir",
-			MultusCNIConfDir:         "/tmp/multus/net.d",
-			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult := `{
-        "cniVersion": "0.3.1",
-        "name": "multus-cni-network",
-        "type": "multus",
-        "namespaceIsolation": true,
-        "globalNamespaces": "foobar,barfoo",
-        "logLevel": "debug",
-        "logFile": "/tmp/foobar.log",
-        "binDir": "/tmp/add_bin_dir",
-        "cniConf": "/tmp/multus/net.d",
-        "readinessindicatorfile": "/var/lib/foobar_indicator",
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
-        ]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), default, conflist", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "1.0.0",
-			"name": "test1",
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult :=
-			`{
-    "cniVersion": "1.0.0",
-    "name": "multus-cni-network",
-    "plugins": [ {
-        "type": "multus",
-        "logToStderr": false,
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
-        ]
-    }]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), capabilities, conflist", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "1.0.0",
-			"name": "test1",
-			"capabilities": { "bandwidth": true },
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult :=
-			`{
-    "cniVersion": "1.0.0",
-    "name": "multus-cni-network",
-    "plugins": [ {
-        "type": "multus",
-        "capabilities": {"bandwidth":true},
-        "logToStderr": false,
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-            {"capabilities":{"bandwidth":true},"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
-        ]
-    }]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), with options, conflist", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfig := `
-		{
-			"cniVersion": "1.0.0",
-			"name": "test1",
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-			NamespaceIsolation:       true,
-			GlobalNamespaces:         "foobar,barfoo",
-			MultusLogToStderr:        true,
-			MultusLogLevel:           "DEBUG",
-			MultusLogFile:            "/tmp/foobar.log",
-			AdditionalBinDir:         "/tmp/add_bin_dir",
-			MultusCNIConfDir:         "/tmp/multus/net.d",
-			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult :=
-			`{
-    "cniVersion": "1.0.0",
-    "name": "multus-cni-network",
-    "plugins": [ {
-        "type": "multus",
-        "namespaceIsolation": true,
-        "globalNamespaces": "foobar,barfoo",
-        "logLevel": "debug",
-        "logFile": "/tmp/foobar.log",
-        "binDir": "/tmp/add_bin_dir",
-        "cniConf": "/tmp/multus/net.d",
-        "readinessindicatorfile": "/var/lib/foobar_indicator",
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
-        ]
-    }]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createMultusConfig(), with options, conflist", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
-		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
-
-		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
-		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
-
-		// create master CNI config
-		masterCNIConfigFileName := "10-testcni.conf"
-		masterCNIConfig := `
-		{
-			"cniVersion": "1.0.0",
-			"name": "test1",
-			"type": "cnitesttype"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, masterCNIConfigFileName), []byte(masterCNIConfig), 0755)).To(Succeed())
-
-		// create another CNI config
-		anotherCNIConfigFileName := "09-test2cni.conf" // Alphabetically before masterCNIConfigFileName
-		anotherCNIConfig := `
-		{
-			"cniVersion": "1.0.0",
-			"name": "test2",
-			"type": "cnitest2type"
-		}`
-		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, anotherCNIConfigFileName), []byte(anotherCNIConfig), 0755)).To(Succeed())
-
-		masterConfigPath, masterConfigHash, err := (&Options{
-			MultusAutoconfigDir:      multusAutoConfigDir,
-			MultusMasterCNIFileName:  masterCNIConfigFileName,
-			CNIConfDir:               cniConfDir,
-			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig(nil)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(masterConfigPath).NotTo(Equal(""))
-		Expect(masterConfigHash).NotTo(Equal(""))
-
-		expectedResult :=
-			`{
-    "cniVersion": "1.0.0",
-    "name": "multus-cni-network",
-    "plugins": [ {
-        "type": "multus",
-        "logToStderr": false,
-        "kubeconfig": "/etc/foobar_kubeconfig",
-        "delegates": [
-            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
-        ]
-    }]
-}
-`
-		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
-		Expect(string(conf)).To(Equal(expectedResult))
-
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-	It("Run createKubeConfig()", func() {
-		// create temp dir and files
-		tmpDir := GinkgoT().TempDir()
-
-		cniConfDir := "/cni_conf"
-		Expect(os.Mkdir(filepath.Join(tmpDir, cniConfDir), 0755)).To(Succeed())
-
-		multusConfDir := "/multus_conf"
-		Expect(os.Mkdir(filepath.Join(tmpDir, multusConfDir), 0755)).To(Succeed())
-
-		// Create service account CA file and token file with dummy data
-		svcAccountPath := filepath.Join(tmpDir, "var/run/secrets/kubernetes.io/serviceaccount")
-		Expect(os.MkdirAll(svcAccountPath, 0755)).ToNot(HaveOccurred())
-		svcAccountCAFile := filepath.Join(tmpDir, serviceAccountCAFile)
-		svcAccountTokenFile := filepath.Join(tmpDir, serviceAccountTokenFile)
-		Expect(os.WriteFile(svcAccountCAFile, []byte("dummy-ca-content"), 0644)).To(Succeed())
-		Expect(os.WriteFile(svcAccountTokenFile, []byte("dummy-token-content"), 0644)).To(Succeed())
-
-		// Set up the Options struct
-		options := &Options{
-			CNIConfDir:       cniConfDir,
-			MultusCNIConfDir: multusConfDir,
-		}
-
-		// Run the createKubeConfig function in a chroot env
-		back, err := chrootTestHelper(tmpDir)
-		Expect(err).ToNot(HaveOccurred())
-		caHash, saTokenHash, err := options.createKubeConfig(nil, nil)
-		Expect(back()).ToNot(HaveOccurred())
-		// back to original root
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(caHash).NotTo(BeNil())
-		Expect(saTokenHash).NotTo(BeNil())
-
-		// Verify the kubeconfig file was created successfully
-		kubeConfigPath := filepath.Join(tmpDir, cniConfDir, "multus.d", "multus.kubeconfig")
-		content, err := os.ReadFile(kubeConfigPath)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(content).NotTo(BeEmpty())
-
-		// Cleanup
-		Expect(os.RemoveAll(tmpDir)).To(Succeed())
-	})
-
-})

deployments/deprecated/multus-daemonset-crio-pre1.16.yml

@@ -0,0 +1,263 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:v3.6
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--override-network-name=true"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--override-network-name=true"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/deprecated/multus-daemonset-gke-pre-1.16.yml

@@ -0,0 +1,232 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+              type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: nfvpe/multus:v3.6
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          # ppc64le support requires multus:latest for now. support 3.3 or later.
+          image: nfvpe/multus:latest-ppc64le
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "90Mi"
+            limits:
+              cpu: "100m"
+              memory: "90Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf

deployments/deprecated/multus-daemonset-pre-1.16.yml

@@ -0,0 +1,249 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: nfvpe/multus:v3.6
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/multus-daemonset-crio.yml

@@ -1,11 +1,3 @@
-# Note:
-#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
-#   hence this deployment yaml does not care about following things intentionally.
-#     - various configuration options
-#     - minor deployment scenario
-#     - upgrade/update/uninstall scenario
-#   Multus team understand users deployment scenarios are diverse, hence we do not cover
-#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -186,6 +178,7 @@ spec:
         - "--cni-version=0.3.1"
         - "--cni-bin-dir=/host/usr/libexec/cni"
         - "--multus-conf-file=auto"
+        - "--restart-crio=true"
         resources:
           requests:
             cpu: "100m"
@@ -197,7 +190,6 @@ spec:
           privileged: true
           capabilities:
             add: ["SYS_ADMIN"]
-        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: run
           mountPath: /run

deployments/multus-daemonset-gke-1.16.yml

@@ -0,0 +1,181 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-version=0.3.1"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf

deployments/multus-daemonset-thick-plugin.yml

@@ -0,0 +1,189 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+          args:
+            - "-cni-version=0.3.1"
+            - "-cni-config-dir=/host/etc/cni/net.d"
+            - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+            - "-multus-log-to-stderr=true"
+            - "-multus-log-level=verbose"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+        - name: generate-kubeconfig
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "/usr/src/multus-cni/bin/generate-kubeconfig"
+          args:
+            - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+            - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+

deployments/multus-daemonset-thick.yml

@@ -1,254 +0,0 @@
-# Note:
-#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
-#   hence this deployment yaml does not care about following things intentionally.
-#     - various configuration options
-#     - minor deployment scenario
-#     - upgrade/update/uninstall scenario
-#   Multus team understand users deployment scenarios are diverse, hence we do not cover
-#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-      - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
-            Working Group to express the intent for attaching pods to one or more logical or physical
-            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
-          type: object
-          properties:
-            apiVersion:
-              description: 'APIVersion defines the versioned schema of this represen
-                tation of an object. Servers should convert recognized schemas to the
-                latest internal value, and may reject unrecognized values. More info:
-                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
-            kind:
-              description: 'Kind is a string value representing the REST resource this
-                object represents. Servers may infer this from the endpoint the client
-                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
-            metadata:
-              type: object
-            spec:
-              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
-              type: object
-              properties:
-                config:
-                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - list
-      - update
-      - watch
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-  - kind: ServiceAccount
-    name: multus
-    namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-daemon-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  daemon-config.json: |
-    {
-        "chrootDir": "/hostroot",
-        "cniVersion": "0.3.1",
-        "logLevel": "verbose",
-        "logToStderr": true,
-        "cniConfigDir": "/host/etc/cni/net.d",
-        "multusAutoconfigDir": "/host/etc/cni/net.d",
-        "multusConfigFile": "auto",
-        "socketDir": "/host/run/multus/"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      hostPID: true
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-        - operator: Exists
-          effect: NoExecute
-      serviceAccountName: multus
-      containers:
-        - name: kube-multus
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
-          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "50Mi"
-            limits:
-              cpu: "100m"
-              memory: "50Mi"
-          securityContext:
-            privileged: true
-          terminationMessagePolicy: FallbackToLogsOnError
-          volumeMounts:
-            - name: cni
-              mountPath: /host/etc/cni/net.d
-            # multus-daemon expects that cnibin path must be identical between pod and container host.
-            # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
-            # not to any other directory, like '/opt/bin' or '/usr/bin'.
-            - name: cnibin
-              mountPath: /opt/cni/bin
-            - name: host-run
-              mountPath: /host/run
-            - name: host-var-lib-cni-multus
-              mountPath: /var/lib/cni/multus
-            - name: host-var-lib-kubelet
-              mountPath: /var/lib/kubelet
-              mountPropagation: HostToContainer
-            - name: host-run-k8s-cni-cncf-io
-              mountPath: /run/k8s.cni.cncf.io
-            - name: host-run-netns
-              mountPath: /run/netns
-              mountPropagation: HostToContainer
-            - name: multus-daemon-config
-              mountPath: /etc/cni/net.d/multus.d
-              readOnly: true
-            - name: hostroot
-              mountPath: /hostroot
-              mountPropagation: HostToContainer
-            - mountPath: /etc/cni/multus/net.d
-              name: multus-conf-dir
-          env:
-            - name: MULTUS_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-      initContainers:
-        - name: install-multus-binary
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
-          command:
-            - "sh"
-            - "-c"
-            - "cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim && cp /usr/src/multus-cni/bin/passthru /host/opt/cni/bin/passthru"
-          resources:
-            requests:
-              cpu: "10m"
-              memory: "15Mi"
-          securityContext:
-            privileged: true
-          terminationMessagePolicy: FallbackToLogsOnError
-          volumeMounts:
-            - name: cnibin
-              mountPath: /host/opt/cni/bin
-              mountPropagation: Bidirectional
-      terminationGracePeriodSeconds: 10
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: hostroot
-          hostPath:
-            path: /
-        - name: multus-daemon-config
-          configMap:
-            name: multus-daemon-config
-            items:
-            - key: daemon-config.json
-              path: daemon-config.json
-        - name: host-run
-          hostPath:
-            path: /run
-        - name: host-var-lib-cni-multus
-          hostPath:
-            path: /var/lib/cni/multus
-        - name: host-var-lib-kubelet
-          hostPath:
-            path: /var/lib/kubelet
-        - name: host-run-k8s-cni-cncf-io
-          hostPath:
-            path: /run/k8s.cni.cncf.io
-        - name: host-run-netns
-          hostPath:
-            path: /run/netns/
-        - name: multus-conf-dir
-          hostPath:
-            path: /etc/cni/multus/net.d

deployments/multus-daemonset.yml

@@ -1,11 +1,3 @@
-# Note:
-#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
-#   hence this deployment yaml does not care about following things intentionally.
-#     - various configuration options
-#     - minor deployment scenario
-#     - upgrade/update/uninstall scenario
-#   Multus team understand users deployment scenarios are diverse, hence we do not cover
-#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -179,12 +171,11 @@ spec:
       serviceAccountName: multus
       containers:
       - name: kube-multus
-        image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
-        command: ["/thin_entrypoint"]
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+        command: ["/entrypoint.sh"]
         args:
         - "--multus-conf-file=auto"
-        - "--multus-autoconfig-dir=/host/etc/cni/net.d"
-        - "--cni-conf-dir=/host/etc/cni/net.d"
+        - "--cni-version=0.3.1"
         resources:
           requests:
             cpu: "100m"
@@ -194,7 +185,6 @@ spec:
             memory: "50Mi"
         securityContext:
           privileged: true
-        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: cni
           mountPath: /host/etc/cni/net.d
@@ -204,18 +194,17 @@ spec:
           mountPath: /tmp/multus-conf
       initContainers:
         - name: install-multus-binary
-          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
-          command: ["/install_multus"]
-          args:
-            - "--type"
-            - "thin"
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
           resources:
             requests:
               cpu: "10m"
               memory: "15Mi"
           securityContext:
             privileged: true
-          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin

docs/configuration.md

@@ -1,16 +1,7 @@
-# Multus-cni Configuration Reference
-
-## Introduction
-
-Aside from setting options for Multus, one of the goals of configuration is to set the configuration for your *default network*. The default network is also sometimes referred as the "primary CNI plugin", the "primary network", or a "default CNI plugin" and is the CNI plugin that is used to implement [the Kubernetes networking model](https://kubernetes.io/docs/concepts/services-networking/#the-kubernetes-network-model) in your cluster. Common examples include Flannel, Weave, Calico, Cillium, and OVN-Kubernetes, among others.
-
-Here we will refer to this as your default CNI plugin or default network.
-
-## Example configuration
+## Multus-cni Configuration Reference
 
 Following is the example of multus config file, in `/etc/cni/net.d/`.
-
-Example configuration using `clusterNetwork` (see also [using delegates](#using-delegates))
+(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
 
 ```
 {
@@ -32,92 +23,14 @@ Example configuration using `clusterNetwork` (see also [using delegates](#using-
     "capabilities": {
         "portMappings": true
     },    
+    "readinessindicatorfile": "",
     "namespaceIsolation": false,
-    "clusterNetwork": "/etc/cni/net.d/99-flannel.conf",
-    "defaultNetworks": ["sidecarCRD", "exampleNetwork"],
+"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
+    "clusterNetwork": "defaultCRD",
+    "defaultNetworks": ["sidecarCRD", "flannel"],
     "systemNamespaces": ["kube-system", "admin"],
     "multusNamespace": "kube-system",
-    "auxiliaryCNIChainName": "cni-chain-config",
-    allowTryDeleteOnErr: false
-}
-```
-
-## Index of configuration options
-
-This is a general index of options, however note that you must set either the `clusterNetwork` or `delegates` options, see the following sections after the index for details.
-
-* `name` (string, required): The name of the network
-* `type` (string, required): Must be set to the value of "multus"
-* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
-* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
-* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
-* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
-* [`logToStderr`](#Logging-via-STDERR) (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
-* [`logFile`](#Writing-to-a-Log-File) (string, optional): file path for log file. multus puts log in given file
-* [`logLevel`](#Logging-Level) (string, optional): logging level (values in decreasing order of verbosity: "debug", "error", "verbose", or "panic")
-* [`logOptions`](#Logging-Options) (object, optional): logging option, More detailed log configuration
-* [`namespaceIsolation`](#Namespace-Isolation) (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
-* [`globalNamespaces`](#Allow-specific-namespaces-to-be-used-across-namespaces-when-using-namespace-isolation): (string, optional): Used only when `namespaceIsolation` is true, allows specification of comma-delimited list of namespaces which may be referred to outside of namespace isolation.
-* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
-* [`readinessindicatorfile`](#Default-Network-Readiness-Indicator): The path to a file whose existence denotes that the default network is ready
-message to next when some missing error. Defaults to false.
-* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
-* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` (the default value is `kube-system`)
-* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL 
-* [`auxiliaryCNIChainName`](#auxiliaryCNIChainName) (string, optional): Enable loading CNI configurations from disk as chained plugins in an auxiliary CNI chain
-
-### Using `clusterNetwork`
-
-Using the `clusterNetwork` option and the `delegates` are **mutually exclusive**. If `clusterNetwork` is set, the `delegates` field is *ignored*. 
-
-You **must** set one or the other.
-
-Therefore:
-
-* Set `clusterNetwork` and if this is set, optionally set the `defaultNetworks`.
-* OR you **must** set `delegates`.
-
-Options:
-
-* `clusterNetwork` (string, required if not using `delegates`): the default CNI plugin to be executed.
-* `defaultNetworks` ([]string, optional): Additional / secondary network attachment that is always attached to each pod. 
-
-The following values are valid for both `clusterNetwork` and `defaultNetworks` and are processed in the following order:
-
-* The name of a `NetworkAttachmentDefinition` custom resource in the namespace specified  by the `multusNamespace` configuration option
-* The `"name"` value in the contents of a CNI JSON configuration file in the CNI configuration directory, 
-  * The given name for `clusterNetwork` should match the value for `name` key in the contents of the CNI JSON file (e.g. `"name": "test"` in `my.conf` when `"clusterNetwork": "test"`)
-* A path to a directory containing CNI json configuration files. The alphabetically first file will be used.
-* Absolute file path for CNI config file
-* If none of the above are found using the value, Multus will raise an error.
-
-If for example you have `defaultNetworks` set as:
-
-```
-"defaultNetworks": ["sidecarNetwork", "exampleNetwork"],
-```
-
-In this example, the values in the expression refer to `NetworkAttachmentDefinition` custom resource names. Therefore, there must be `NetworkAttachmentDefinitions` already created with the names `sidecarNetwork` and `exampleNetwork`.
-
-This means that in addition to the cluster network, each pod would be assigned two additional networks by default, and the pod would present three interfaces, e.g. `eth0`, `net1`, and `net2`, with `net1` and `net2` being set by the above described `NetworkAttachmentDefinitions`. Additional attachments as made by setting `k8s.v1.cni.cncf.io/networks` on pods will be made in addition to those set in the `defaultNetworks` configuration option.
-
-### Using `delegates`
-
-If `clusterNetwork` is not set, you **must** use `delegates`.
-
-* `delegates` ([]map, required if not using `clusterNetwork`). List of CNI configurations to be used as your default CNI plugin(s).
-
-Example configuration using `delegates`:
-
-```
-{
-    "cniVersion": "0.3.1",
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-    "confDir": "/etc/cni/multus/net.d",
-    "cniDir": "/var/lib/cni/multus",
-    "binDir": "/opt/cni/bin",
+"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
     "delegates": [{
         "type": "weave-net",
         "hairpinMode": true
@@ -128,13 +41,44 @@ Example configuration using `delegates`:
 }
 ```
 
-## Configuration Option Details
+* `name` (string, required): the name of the network
+* `type` (string, required): "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
+* `logFile` (string, optional): file path for log file. multus puts log in given file
+* `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
+* `logOptions` (object, optional): logging option, More detailed log configuration
+* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
+
+User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
+
+* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
+* `delegates` ([]map,required): number of delegate details in the Multus
+
+### Network selection flow of clusterNetwork/defaultNetworks
+
+Multus will find network for clusterNetwork/defaultNetworks as following sequences:
+
+1. CRD object for given network name, in 'kube-system' namespace
+1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
+1. Directory for CNI json config file. Multus will find alphabetically first file for the network
+1. Multus failed to find network. Multus raise error message
+
+## Miscellaneous config
 
 ### Default Network Readiness Indicator
 
-You may desire that your default network becomes ready before attaching networks with Multus. This is disabled by default and not used unless you set the `readinessindicatorfile` option to a non-blank value.
+You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
 
-For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, therefore, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
 
 In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
 
@@ -382,47 +326,3 @@ annotations:
  v1.multus-cni.io/default-network: calico-conf
 ...
 ```
-
-### `auxiliaryCNIChainName`
-
-`auxiliaryCNIChainName` (of value string) is used to express the name of an additional auxiliary CNI chain that will execute in order to composably execute chained CNI plugins from configurations on the host's disk in a subdirectory of the CNI configuration directory.
-
-**NOTE**: The path used to determine the base for the subdirectory is the pathname of the `clusterNetwork` value, which must be set to a file in order to use this functionality.
-
-When this string is set, Multus will execute an additional CNI chain, outside of the default network, on its own independent CNI chain (as to not interfere with default network functionality that might be hampered by CNI chaining and to otherwise isolate this execution) and will load CNI configurations from a subdirectory of the same name in the CNI configuration directory.
-
-This feature is based on [improvements made to libcni for "safe subdirectory-based plugin conf loading"](https://github.com/containernetworking/cni/pull/1052). 
-
-`auxiliaryCNIChainName` is meant to be set as a CNI configuration name, this name is arbitrary but must match the subdirectory name.
-
-Consider this [daemon configuration](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/deployments/multus-daemonset-thick.yml#L113):
-
-```
-{
-  "cniConfigDir": "/host/etc/cni/net.d",
-  "multusAutoconfigDir": "/host/etc/cni/net.d",
-  "multusConfigFile": "auto",
-  "socketDir": "/host/run/multus/",
-  "auxiliaryCNIChainName": "cni-chain-config"
-}
-```
-
-Here we have set `"auxiliaryCNIChainName": "cni-chain-config"`, and we have expressed that our CNI configurations are on `/etc/cni/net.d/` on the host.
-
-In this case, we would also have a directory named in `/etc/cni/net.d/cni-chain-config`
-
-One could add any number of CNI configurations to be used as part of this chain, consider this example if we added a tuning CNI configuration called `/etc/cni/net.d/cni-chain-config/mytuning.conf` with these contents:
-
-```
-{
-  "name": "mytuning",
-  "type": "tuning",
-  "sysctl": {
-    "net.ipv4.conf.IFNAME.arp_filter": "1"
-  }
-}
-```
-
-With the given configuration, plus this configuration, this would be executed for every pod launched by Multus CNI.
-
-If this is unset, no auxiliary chain will be executed. However, if the default network CNI configuration is loaded from disk and is a conflist format, the libcni functionality for loading from a subdirectory will still apply.

docs/development.md

@@ -1,14 +1,4 @@
-## Development/Support Information
-
-## Which Kubernetes version is supported in multus?
-
-Currently multus team supports Kubernetes that Kubernetes community maintains.
-See [Version Skew Policy](https://kubernetes.io/releases/version-skew-policy/) for the details.
-
-## How to debug multus-cni thin image?
-
-Latest multus uses [distroless](https://github.com/GoogleContainerTools/distroless) container image for its base,
-hence there is no shell command. If you want to execute shell in multus pod, please use `-debug` image (e.g. ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-debug), which has shell.
+## Development Information
 
 ## How to utilize multus-cni code as library?
 
@@ -16,9 +6,10 @@ Multus now uses [gopkg.in](http://gopkg.in/) to expose its code as library.
 You can use following command to import our code into your go code.
 
 ```
-go get gopkg.in/k8snetworkplumbingwg/multus-cni.v4
+go get gopkg.in/k8snetworkplumbingwg/multus-cni.v3
 ```
 
+
 ## How do I submit an issue?
 
 Use GitHub as normally, you'll be presented with an option to submit a issue or enhancement request.
@@ -39,7 +30,7 @@ cd multus-cni
 ./hack/build-go.sh
 ```
 
-## How do I run the unit tests?
+## How do I run CI tests?
 
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
@@ -47,10 +38,6 @@ Multus has go unit tests (based on ginkgo framework).The following commands driv
 sudo ./hack/test-go.sh
 ```
 
-## How do I run the e2e tests?
-
-Check the `README.md` in the `./e2e/` folder.
-
 ## What are the best practices for logging?
 
 The following are the best practices for multus logging:
@@ -63,7 +50,3 @@ The following are the best practices for multus logging:
 ## Multus release schedule
 
 On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.
-
-## Multi-arch builds
-
-Multus is currently built for a number of architectures, however, our testing and validation is only performed against x86 architectures. Our x86 architecture has end to end testing, however, for other architectures, only supported via best effort community contributions.

docs/how-to-use.md

@@ -19,13 +19,9 @@ You may acquire the Multus binary via compilation (see the [developer guide](dev
 
 *Via Daemonset method*
 
-As a [quickstart](quickstart.md), you may apply these YAML files. Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
+As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml  # thin deployment
-
-or
-
-    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml # thick (client/server) deployment
+    cat ./deployments/multus-daemonset.yml | kubectl apply -f -
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -43,7 +39,7 @@ cat >/etc/cni/net.d/00-multus.conf <<EOF
 {
   "name": "multus-cni-network",
   "type": "multus",
-  "readinessindicatorfile": "/run/flannel/subnet.env",
+  "readinessindicatorfile": "/var/run/flannel/subnet.env",
   "delegates": [
     {
       "NOTE1": "This is example, wrote your CNI config in delegates",
@@ -126,7 +122,7 @@ Create kubeconfig at master node as following commands:
 mkdir -p /etc/cni/net.d/multus.d
 SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
 SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-KUBERNETES_SERVICE_PROTOCOL=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
 KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
 KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
 cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
@@ -264,7 +260,6 @@ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
       ]
   }
 }
-EOF
 ```
 
 ### Run pod with network annotation
@@ -511,7 +506,7 @@ spec:
 EOF
 ```
 
-We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation.
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation. 
 
 ```
 cat <<EOF | kubectl create -f -
@@ -537,9 +532,9 @@ This will set `192.168.2.1` as the default route over the `net1` interface, such
 ```
 kubectl exec -it samplepod -- ip route
 
-default via 192.168.2.1 dev net1
-10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169
-10.244.0.0/16 via 10.244.0.1 dev eth0
+default via 192.168.2.1 dev net1 
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
+10.244.0.0/16 via 10.244.0.1 dev eth0 
 ```
 
 ## Entrypoint Parameters
@@ -551,7 +546,7 @@ Typically, you'd modified the daemonset YAML itself to specify these parameters.
 For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
 
 ```
-  command: ["/thin_entrypoint"]
+  command: ["/entrypoint.sh"]
   args:
   - "--multus-conf-file=auto"
   - "--namespace-isolation=true"
@@ -590,7 +585,7 @@ The `--multus-conf-file` is one of two options; it can be set to a source file t
 
 The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
 
-In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will take the alphabetically first configuration there and wrap that into a Multus configuration.
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
 
     --multus-autoconfig-dir=/host/etc/cni/net.d
 
@@ -619,13 +614,9 @@ In some cases, the original CNI configuration that the Multus configuration was
 
     --cleanup-config-on-exit=true
 
-When specifying `--cleanup-config-on-exit=true` the entrypoint script will delete any generated/copied Multus configuration files when entrypoint script
-exits (upon Pod termination). This allows Multus to be safely removed from the cluster when its no longer needed.
+When using CRIO, you may need to restart CRIO to get the Multus configuration file to take -- this is rarely necessary.
 
-In addition, when both `--cleanup-config-on-exit=true` and `--multus-conf-file=auto` are specified, the entrypoint script will watch for changes of the
-master CNI configuration and kubeconfig. when such change detected, the script will re-genereate Multus configuration. Watch can be skipped by setting:
-
-    --skip-config-watch
+    --restart-crio=false
 
 Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
 
@@ -642,126 +633,3 @@ Sometimes, you may wish to not have the entrypoint copy the binary file onto the
 If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
 
     --readiness-indicator-file=/path/to/file
-
-### Run pod with network annotation and Dynamic Resource Allocation driver
-
-> :warning: Dynamic Resource Allocation (DRA) is [currently an alpha](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/),
-> and is subject to change. Please consider this functionality as a preview. The architecture and usage of DRA in
-> Multus CNI may change in the future as this technology matures.
->
-> The current DRA integration is based on the DRA API for Kubernetes 1.26 to 1.30. With Kubernetes 1.31, the DRA API
-> will change and multus doesn't integrate with the new API yet.
-
-Dynamic Resource Allocation is alternative mechanism to device plugin which allows to requests pod and container
-resources.
-
-The following sections describe how to use DRA with multus and NVIDIA DRA driver. Other DRA networking driver vendors
-should follow similar concepts to make use of multus DRA support.
-
-#### Prerequisite
-
-1. Kubernetes 1.27
-2. Container Runtime with CDI support enabled
-3. Kubernetes runtime-config=resource.k8s.io/v1alpha2
-4. Kubernetes feature-gates=DynamicResourceAllocation=True,KubeletPodResourcesDynamicResources=true
-
-#### Install DRA driver
-
-The current example uses NVIDIA DRA driver for networking. This DRA driver is not publicly available. An alternative to
-this DRA driver is available at [dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver).
-
-#### Create dynamic resource class with NVIDIA network DRA driver
-
-The `ResourceClass` defines the resource pool of `sf-pool-1`.
-
-```
-# Execute following command at Kubernetes master
-cat <<EOF | kubectl create -f -
-apiVersion: resource.k8s.io/v1alpha2
-kind: ResourceClass
-metadata:
-  name: sf-pool-1
-driverName: net.resource.nvidia.com
-EOF
-```
-
-#### Create network attachment definition with resource name
-
-The `k8s.v1.cni.cncf.io/resourceName` should match the `ResourceClass` name defined in the section above.
-In this example it is `sf-pool-1`. Multus query the K8s PodResource API to fetch the `resourceClass` name and also
-query the NetworkAttachmentDefinition `k8s.v1.cni.cncf.io/resourceName`. If both has the same name multus send the
-CDI device name in the DeviceID argument.
-
-##### NetworkAttachmentDefinition for ovn-kubernetes example:
-
-Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
-
-```
-# Execute following command at Kubernetes master
-cat <<EOF | kubectl create -f -
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: default
-  annotations:
-    k8s.v1.cni.cncf.io/resourceName: sf-pool-1
-spec:
-  config: '{
-      "cniVersion": "0.4.0",
-      "dns": {},
-      "ipam": {},
-      "logFile": "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
-      "logLevel": "4",
-      "logfile-maxage": 5,
-      "logfile-maxbackups": 5,
-      "logfile-maxsize": 100,
-      "name": "ovn-kubernetes",
-      "type": "ovn-k8s-cni-overlay"
-    }'
-EOF
-```
-
-#### Create DRA Resource Claim
-
-Following command creates `ResourceClaim` `sf` which request resource from  `ResourceClass` `sf-pool-1`.
-
-```
-# Execute following command at Kubernetes master
-cat <<EOF | kubectl create -f -
-apiVersion: resource.k8s.io/v1alpha2
-kind: ResourceClaim
-metadata:
-  namespace: default
-  name: sf
-spec:
-  spec:
-    resourceClassName: sf-pool-1
-EOF
-```
-
-#### Launch pod with DRA Resource Claim
-
-Following command Launch a Pod with primiry network `default` and `ResourceClaim` `sf`.
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  namespace: default
-  name: test-sf-claim
-  annotations:
-    v1.multus-cni.io/default-network: default
-spec:
-  restartPolicy: Always
-  containers:
-  - name: with-resource
-    image: docker.io/library/ubuntu:22.04
-    command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"]
-    resources:
-      claims:
-      - name: resource
-  resourceClaims:
-  - name: resource
-    source:
-      resourceClaimName: sf
-```

docs/quickstart.md

@@ -42,19 +42,16 @@ master-2              Ready    master          1h     v1.17.1
 
 Our recommended quickstart method to deploy Multus is to deploy using a Daemonset (a method of running pods on each nodes in your cluster), this spins up pods which install a Multus binary and configure Multus for usage.
 
-We'll apply a YAML file with `kubectl` from this repo, which installs the Multus components.
-
-Recommended installation:
+Firstly, clone this GitHub repository. 
 
 ```
-kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
 ```
-See the [thick plugin docs](./thick-plugin.md) for more information about this architecture.
 
-Alternatively, you may install the thin-plugin with:
+We'll apply a YAML file with `kubectl` from this repo.
 
 ```
-kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
 ### What the Multus daemonset does

docs/thick-plugin.md

@@ -1,114 +0,0 @@
-# Multus Thick plugin
-
-Multus CNI can also be deployed using a thick plugin architecture, which is
-characterized by a client/server architecture.
-
-The client - which will be referred to as "shim" - is a binary executable
-located on the Kubernetes node's file-system that
-[speaks CNI](https://github.com/containernetworking/cni/blob/master/SPEC.md#section-2-execution-protocol):
-the runtime - Kubernetes - passes parameters to the plugin via environment
-variables and configuration - which is passed via stdin.
-The plugin returns a result on stdout on success, or an error on stderr if the
-operation fails. Configuration and results are a JSON encoded string.
-
-Once the shim is invoked by the runtime (Kubernetes) it will contact the
-multus-daemon (server) via a unix domain socket which is bind mounted to the
-host's file-system; the multus-daemon is the one that will do all the
-heavy-pulling: fetch the delegate CNI configuration from the corresponding
-`net-attach-def`, compute the `RuntimeConfig`, and finally, invoke the delegate.
-
-It will then return the result of the operation back to the client.
-
-Please refer to the diagram below for a visual representation of the flow
-described above:
-
-```
-┌─────────┐             ┌───────┐           ┌────────┐             ┌──────────┐
-│         │ cni ADD/DEL │       │ REST POST │        │ cni ADD/DEL │          │
-│ runtime ├────────────►│ shim  │===========│ daemon ├────────────►│ delegate │
-│         │<------------│       │           │        │<------------│          │
-└─────────┘             └───────┘           └────────┘             └──────────┘
-```
-
-## How to use it
-
-### Configure Deployment
-
-If your delegate CNI plugin requires some files which is in container host, please update
-update `deployments/multus-daemonset-thick.yml` to add directory into multus-daemon pod.
-For example, flannel requires `/run/flannel/subnet.env`, so you need to mount this directory
-into the multus-daemon pod.
-
-Required directory/files are different for each CNI plugin, so please refer your CNI plugin.
-
-### Deployment
-
-There is a dedicated multus daemonset specification for users wanting to use
-this thick plugin variant. This reference deployment spec of multus can be
-deployed by following these commands:
-
-```bash
-kubectl apply -f deployments/multus-daemonset-thick.yml
-```
-
-### Command line parameters
-
-The available command line parameters are:
-
-- `config`: Defaults to `"/etc/cni/net.d/multus.d/daemon-config.json"`
-- `version`: Prints the daemon config version and exits
-
-### Server / Daemon configuration
-
-The server configuration is encoded in JSON, and allows the following keys:
-
-- `"chrootDir"`: Specify the directory which points to host root from the pod. See 'Chroot configuration' section for the details.
-- `"socketDir"`: Specify the location where the unix domain socket used
-for client/server communication will be located. This is the location where the
-**Daemon** will read the configuration from. Defaults to `"/run/multus"`.
-- `"metricsPort"`: Metrics port (of multus' metric exporter); by default, no port
-is provided.
-- `"logFile"`: the path to where the daemon logs will be persisted.
-- `"logLevel"`: the logging level for the multus daemon logs.
-- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
-as well. By default, it is disabled.
-- `"auxiliaryCNIChainName"`: set a value to execute chained cni configurations from disk in an auxiliary CNI chain (see details in [configuration.md](configuration.md))
-
-In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
-
-Below you can see an example of the daemon configuration:
-```json
-{
-        "chrootDir": "/hostroot",
-        "confDir": "/host/etc/cni/net.d",
-        "logToStderr": true,
-        "logLevel": "verbose",
-        "logFile": "/tmp/multus.log",
-        "binDir": "/opt/cni/bin",
-        "cniDir": "/var/lib/cni/multus",
-        "socketDir": "/host/run/multus/",
-        "cniVersion": "0.3.1",
-        "cniConfigDir": "/host/etc/cni/net.d",
-        "multusConfigFile": "auto",
-        "multusAutoconfigDir": "/host/etc/cni/net.d"
-    }
-```
-
-### Client / Shim configuration
-
-The multus shim configuration is encoded in JSON, and essentially is just a
-regular CNI configuration, usually available in `/etc/cni/net.d/00-multus.conf`.
-
-It allows the following keys:
-
-- `"cniVersion"`: the CNI version for the Multus CNI plugin.
-- `"logFile"`:  the path to where the daemon logs will be persisted.
-- `"logLevel"`: the logging level for the multus daemon logs.
-- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
-  as well. By default, it is disabled.
-
-#### Chroot configuration
-
-In thick plugin case, delegate CNI plugin is executed by multus-daemon from Pod, hence if the delegate CNI requires resources in container host, for example unix socket or even file, then CNI plugin is failed to execute because multus-daemon runs in Pod. Multus-daemon supports "chrootDir" option which executes delegate CNI under chroot (to container host).
-
-This configuration is enabled in deployments/multus-daemonset-thick.yml as default.

e2e/README.md

@@ -1,36 +1,12 @@
 ## Multus e2e test with kind
 
-### Prerequisite
-
-To run the e2e test, you need the following components:
-
-- curl
-- jinjanator (optional)
-- docker
-
 ### How to test e2e
 
+
 ```
 $ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 $ cd multus-cni/e2e
 $ ./get_tools.sh
-```
-
-If you have `jinjanator` you can generate the YAML with:
-
-```
-$ ./generate_yamls.sh
-```
-
-Alternatively, if you have trouble with it, use the `sed` script.
-
-```
-$ ./e2e/sed_generate_yaml.sh
-```
-
-Then, setup the cluster
-
-```
 $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```

e2e/cni-install.yml

@@ -0,0 +1,64 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: cni-install-sh
+  namespace: kube-system
+data:
+  install_cni.sh: |
+    cd /tmp
+    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
+    cd /host/opt/cni/bin
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
+    sleep infinite
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: install-cni-plugins
+  namespace: kube-system
+  labels:
+    name: cni-plugins
+spec:
+  selector:
+    matchLabels:
+      name: cni-plugins
+  template:
+    metadata:
+      labels:
+        name: cni-plugins
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: install-cni-plugins
+        image: alpine
+        command: ["/bin/sh", "/scripts/install_cni.sh"]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: scripts
+          configMap:
+            name: cni-install-sh
+            items:
+            - key: install_cni.sh
+              path: install_cni.sh

e2e/default-route1.yml

@@ -0,0 +1,57 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default-route-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config",
+              "ips": [ "10.1.1.21/24" ] ,
+              "default-route": [ "10.1.1.254" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config", 
+              "ips": [ "10.1.1.22/24" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/generate_yamls.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-if [ ! -d yamls ]; then
-	mkdir yamls
-fi
-
-# specify CNI version (default: 0.4.0)
-export CNI_VERSION=${CNI_VERSION:-0.4.0}
-
-templates_dir="$(dirname $(readlink -f $0))/templates"
-
-# generate yaml files based on templates/*.j2 to yamls directory
-for i in `ls templates/`; do
-	echo $i
-	j2 -e CNI_VERSION ${templates_dir}/$i -o yamls/${i%.j2}
-done
-unset CNI_VERSION

e2e/get_tools.sh

@@ -5,7 +5,7 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.27.0/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl
@@ -13,4 +13,3 @@ curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq
-wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm

e2e/legacy-multus-daemonset.yml

@@ -0,0 +1,264 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/multus-daemonset.yml

@@ -0,0 +1,296 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        imagePullPolicy: Always
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+        args:
+        - "-multus-conf-file=auto"
+        - "-cni-version=0.3.1"
+        - "-cni-config-dir=/host/etc/cni/net.d"
+        - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "-multus-log-to-stderr=true"
+        - "-multus-log-level=debug"
+        - "-multus-log-file=/tmp/multus.log"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      - name: generate-kubeconfig
+        image: localhost:5000/multus:e2e
+        command:
+          - "/usr/src/multus-cni/bin/generate-kubeconfig"
+        args:
+          - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+          - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cni
+            mountPath: /host/etc/cni/net.d
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/sed_generate_yaml.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-if [ ! -d yamls ]; then
-	mkdir yamls
-fi
-
-# specify CNI version (default: 0.4.0)
-CNI_VERSION=${CNI_VERSION:-0.4.0}
-
-templates_dir="$(dirname $(readlink -f $0))/templates"
-
-# generate yaml files based on templates/*.j2 to yamls directory
-for i in `ls ${templates_dir}/*.j2`; do
-	echo "Processing $i..."
-	# Use sed to replace the placeholder with the CNI_VERSION variable
-	sed "s/{{ CNI_VERSION }}/$CNI_VERSION/g" $i > yamls/$(basename ${i%.j2})
-done

e2e/setup_cluster.sh

@@ -8,42 +8,36 @@ export PATH=${PATH}:./bin
 OCI_BIN="${OCI_BIN:-docker}"
 
 # define the deployment spec to use when deploying multus.
-# Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
-# Defaults to `multus-daemonset-thick.yml`.
-MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
-# define the dockerfile to build multus.
-# Acceptable values are `Dockerfile`. `Dockerfile.thick`.
-# Defaults to `Dockerfile.thick`.
-MULTUS_DOCKERFILE="${MULTUS_DOCKERFILE:-Dockerfile.thick}"
+# Acceptable values are `legacy-multus-daemonset.yml`. `multus-daemonset.yml`.
+# Defaults to `multus-daemonset.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset.yml}"
 
 kind_network='kind'
-if [ "${MULTUS_DOCKERFILE}" != "none" ]; then
-	$OCI_BIN build -t localhost:5000/multus:e2e -f ../images/${MULTUS_DOCKERFILE} ..
+reg_name='kind-registry'
+reg_port='5000'
+running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  # run registry and push the multus image
+  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
+  $OCI_BIN push localhost:5000/multus:e2e
 fi
+reg_host="${reg_name}"
+if [ "${kind_network}" = "bridge" ]; then
+    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+fi
+echo "Registry Host: ${reg_host}"
 
 # deploy cluster with kind
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_host}:${reg_port}"]
 nodes:
   - role: control-plane
-    kubeadmConfigPatches:
-    - |
-      kind: ClusterConfiguration
-      apiServer:
-        extraArgs:
-          runtime-config: "resource.k8s.io/v1beta1=true"
-      scheduler:
-        extraArgs:
-          v: "1"
-      controllerManager:
-        extraArgs:
-          v: "1"
-    - |
-      kind: InitConfiguration
-      nodeRegistration:
-        kubeletExtraArgs:
-          v: "1"
   - role: worker
     kubeadmConfigPatches:
     - |
@@ -51,44 +45,31 @@ nodes:
       nodeRegistration:
         kubeletExtraArgs:
           pod-manifest-path: "/etc/kubernetes/manifests/"
-          feature-gates: "DynamicResourceAllocation=true,DRAResourceClaimDeviceStatus=true,KubeletPodResourcesDynamicResources=true"
-    - |
-      kind: JoinConfiguration
-      nodeRegistration:
-        kubeletExtraArgs:
-          v: "1"
   - role: worker
-    kubeadmConfigPatches:
-    - |
-      kind: InitConfiguration
-      nodeRegistration:
-        kubeletExtraArgs:
-          pod-manifest-path: "/etc/kubernetes/manifests/"
-          feature-gates: "DynamicResourceAllocation=true,DRAResourceClaimDeviceStatus=true,KubeletPodResourcesDynamicResources=true"
-    - |
-      kind: JoinConfiguration
-      nodeRegistration:
-        kubeletExtraArgs:
-          v: "1"
-# Required by DRA Integration
-##
-featureGates:
-  DynamicResourceAllocation: true
-  DRAResourceClaimDeviceStatus: true
-  KubeletPodResourcesDynamicResources: true
-runtimeConfig:
-  "api/beta": "true"
-containerdConfigPatches:
-# Enable CDI as described in
-# https://github.com/container-orchestrated-devices/container-device-interface#containerd-configuration
-- |-
-  [plugins."io.containerd.grpc.v1.cri"]
-      enable_cdi = true
-##
 EOF
 
-# load multus image from container host to kind node
-kind load docker-image localhost:5000/multus:e2e
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
+needs_connect="true"
+for c in $containers; do
+  if [ "$c" = "${reg_name}" ]; then
+    needs_connect="false"
+  fi
+done
+if [ "${needs_connect}" = "true" ]; then
+  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
+fi
 
 worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
 worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
@@ -97,9 +78,9 @@ kind export kubeconfig
 sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
 sleep 1
 kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
-kubectl create -f yamls/$MULTUS_MANIFEST
+kubectl create -f "$MULTUS_MANIFEST"
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
-kubectl create -f yamls/cni-install.yml
+kubectl create -f cni-install.yml
 sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/simple-macvlan1.yml

@@ -0,0 +1,63 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan1-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "capabilities": { "ips": true },
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                }, {
+                    "type": "tuning"
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.11/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.12/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker2

e2e/teardown.sh

@@ -1,7 +1,10 @@
 #!/bin/sh
 #set -o errexit
 
+reg_name='kind-registry'
 export PATH=${PATH}:./bin
 
 # delete cluster kind
 kind delete cluster
+docker kill ${reg_name}
+docker rm ${reg_name}

e2e/templates/cni-install.yml.j2

@@ -1,64 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: cni-install-sh
-  namespace: kube-system
-data:
-  install_cni.sh: |
-    cd /tmp
-    wget https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz
-    cd /host/opt/cni/bin
-    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.4.0.tgz
-    sleep infinite
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: install-cni-plugins
-  namespace: kube-system
-  labels:
-    name: cni-plugins
-spec:
-  selector:
-    matchLabels:
-      name: cni-plugins
-  template:
-    metadata:
-      labels:
-        name: cni-plugins
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: install-cni-plugins
-        image: alpine
-        command: ["/bin/sh", "/scripts/install_cni.sh"]
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni-bin
-          mountPath: /host/opt/cni/bin
-        - name: scripts
-          mountPath: /scripts
-      volumes:
-        - name: cni-bin
-          hostPath:
-            path: /opt/cni/bin
-        - name: scripts
-          configMap:
-            name: cni-install-sh
-            items:
-            - key: install_cni.sh
-              path: install_cni.sh

e2e/templates/default-route1.yml.j2

@@ -1,57 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: default-route-config
-spec: 
-  config: '{
-            "cniVersion": "{{ CNI_VERSION }}",
-            "plugins": [
-                {
-                    "type": "macvlan",
-                    "master": "eth1",
-                    "mode": "bridge",
-                    "ipam": {
-                        "type": "static"
-                    }
-                } ]
-        }'
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: default-route-worker1
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "default-route-config",
-              "ips": [ "10.1.1.21/24" ] ,
-              "default-route": [ "10.1.1.254" ] }
-    ]'
-  labels:
-    app: default-route1
-spec:
-  containers:
-  - name: default-route-worker1
-    image: centos:8
-    command: ["/bin/sleep", "10000"]
-    securityContext:
-      privileged: true
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: default-route-worker2
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "default-route-config", 
-              "ips": [ "10.1.1.22/24" ] }
-    ]'
-  labels:
-    app: default-route1
-spec:
-  containers:
-  - name: default-route-worker2
-    image: centos:8
-    command: ["/bin/sleep", "10000"]
-    securityContext:
-      privileged: true

e2e/templates/dra-integration.yml.j2

@@ -1,51 +0,0 @@
----
-apiVersion: resource.k8s.io/v1beta1
-kind: ResourceClaimTemplate
-metadata:
-  name: single-gpu
-spec:
-  spec:
-    devices:
-      requests:
-      - name: gpu
-        deviceClassName: gpu.example.com
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: dra-net
-  annotations:
-    k8s.v1.cni.cncf.io/resourceName: single-gpu
-spec:
-  config: '{
-        "cniVersion": "{{ CNI_VERSION }}",
-        "plugins": [{
-            "name": "mynet",
-            "type": "dummy",
-            "ipam": {
-                "type": "host-local",
-                "subnet": "10.1.2.0/24"
-            }
-        }]
-    }'
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: dra-integration
-  labels:
-    app: dra-integration
-  annotations:
-    k8s.v1.cni.cncf.io/networks: default/dra-net
-spec:
-  containers:
-  - name: ctr0
-    image: ubuntu:22.04
-    command: ["bash", "-c"]
-    args: ["export; sleep 9999"]
-    resources:
-      claims:
-      - name: gpu
-  resourceClaims:
-  - name: gpu
-    resourceClaimTemplateName: single-gpu

e2e/templates/multus-daemonset-thick.yml.j2

@@ -1,210 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              properties:
-                config:
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - list
-      - update
-      - watch
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-daemon-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  daemon-config.json: |
-    {
-        "confDir": "/host/etc/cni/net.d",
-        "logToStderr": true,
-        "logLevel": "debug",
-        "logFile": "/tmp/multus.log",
-        "binDir": "/host/opt/cni/bin",
-        "cniDir": "/var/lib/cni/multus",
-        "socketDir": "/host/run/multus",
-        "cniVersion": "{{ CNI_VERSION }}",
-        "cniConfigDir": "/host/etc/cni/net.d",
-        "multusConfigFile": "auto",
-        "forceCNIVersion": true,
-        "multusAutoconfigDir": "/host/etc/cni/net.d"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      hostPID: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: localhost:5000/multus:e2e
-        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: host-run
-          mountPath: /host/run
-        - name: host-var-lib-cni-multus
-          mountPath: /var/lib/cni/multus
-        - name: host-run-netns
-          mountPath: /run/netns
-          mountPropagation: HostToContainer
-        - name: multus-daemon-config
-          mountPath: /etc/cni/net.d/multus.d
-          readOnly: true
-        - name: kubelet-pod-resources
-          mountPath: /var/lib/kubelet/pod-resources
-          readOnly: true
-        env:
-        - name: MULTUS_NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-      initContainers:
-      - name: install-multus-shim
-        image: localhost:5000/multus:e2e
-        command:
-          - "sh"
-          - "-c"
-          - "cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim && cp /usr/src/multus-cni/bin/passthru /host/opt/cni/bin/passthru"
-        resources:
-          requests:
-            cpu: "10m"
-            memory: "15Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-          - name: cnibin
-            mountPath: /host/opt/cni/bin
-            mountPropagation: Bidirectional
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: kubelet-pod-resources
-          hostPath:
-            path: /var/lib/kubelet/pod-resources
-        - name: multus-daemon-config
-          configMap:
-            name: multus-daemon-config
-            items:
-            - key: daemon-config.json
-              path: daemon-config.json
-        - name: host-run
-          hostPath:
-            path: /run
-        - name: host-var-lib-cni-multus
-          hostPath:
-            path: /var/lib/cni/multus
-        - name: host-run-netns
-          hostPath:
-            path: /run/netns/

e2e/templates/multus-daemonset.yml.j2

@@ -1,198 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  versions:
-    - name: v1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              properties:
-                config:
-                  type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-    name: multus
-spec:
-  selector:
-    matchLabels:
-      name: multus
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-        name: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        image: localhost:5000/multus:e2e
-        command: ["/thin_entrypoint"]
-        args:
-        - "--multus-conf-file=auto"
-        - "--force-cni-version=true"
-        - "--cni-version={{ CNI_VERSION }}"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/opt/cni/bin
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      initContainers:
-      - name: install-multus-binary
-        image: localhost:5000/multus:e2e
-        command: ["/install_multus"]
-        args:
-          - "--type"
-          - "thin"
-        resources:
-          requests:
-            cpu: "10m"
-            memory: "15Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-          - name: cnibin
-            mountPath: /host/opt/cni/bin
-            mountPropagation: Bidirectional
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /opt/cni/bin
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf

e2e/templates/simple-macvlan1.yml.j2

@@ -1,63 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan1-config
-spec: 
-  config: '{
-            "cniVersion": "{{ CNI_VERSION }}",
-            "plugins": [
-                {
-                    "type": "macvlan",
-                    "capabilities": { "ips": true },
-                    "master": "eth1",
-                    "mode": "bridge",
-                    "ipam": {
-                        "type": "static"
-                    }
-                }, {
-                    "type": "tuning"
-                } ]
-        }'
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: macvlan1-worker1
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan1-config",
-              "ips": [ "10.1.1.11/24" ] }
-    ]'
-  labels:
-    app: macvlan
-spec:
-  containers:
-  - name: macvlan-worker1
-    image: centos:8
-    command: ["/bin/sleep", "10000"]
-    securityContext:
-      privileged: true
-  nodeSelector:
-    kubernetes.io/hostname: kind-worker
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: macvlan1-worker2
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan1-config",
-              "ips": [ "10.1.1.12/24" ] }
-    ]'
-  labels:
-    app: macvlan
-spec:
-  containers:
-  - name: macvlan-worker2
-    image: centos:8
-    command: ["/bin/sleep", "10000"]
-    securityContext:
-      privileged: true
-  nodeSelector:
-    kubernetes.io/hostname: kind-worker2

e2e/templates/subdirectory-chain-passthru-configupdate.yml.j2

@@ -1,26 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-daemon-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  daemon-config.json: |
-    {
-        "confDir": "/host/etc/cni/net.d",
-        "logToStderr": true,
-        "logLevel": "debug",
-        "logFile": "/tmp/multus.log",
-        "binDir": "/host/opt/cni/bin",
-        "cniDir": "/var/lib/cni/multus",
-        "socketDir": "/host/run/multus",
-        "cniVersion": "{{ CNI_VERSION }}",
-        "cniConfigDir": "/host/etc/cni/net.d",
-        "multusConfigFile": "auto",
-        "forceCNIVersion": true,
-        "multusAutoconfigDir": "/host/etc/cni/net.d",
-        "auxiliaryCNIChainName": "vendor-cni-chain"
-    }

e2e/templates/subdirectory-chaining-passthru.yml.j2

@@ -1,94 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cni-setup-script
-  namespace: default
-data:
-  setup.sh: |
-    #!/bin/bash
-    set -euxo pipefail
-
-    DEFAULT_NETWORK_CNI_NAME="vendor-cni-chain"
-
-    cleanup() {
-      echo "Cleaning up..."
-      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
-      if [ $? -ne 0 ]; then
-        echo "Failed to remove sysctltwiddle.conf" >&2
-        exit 1
-      fi
-      echo "Cleanup completed successfully"
-    }
-    trap cleanup EXIT
-
-    # Create the chained CNI directory if it doesn't exist
-    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
-    if [ $? -ne 0 ]; then
-      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
-      exit 1
-    fi
-
-    # Write the chained tuning CNI config
-    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
-    {
-      "cniVersion": "{{ CNI_VERSION }}",
-      "name": "sysctltwiddle",
-      "type": "tuning",
-      "sysctl": {
-        "net.ipv4.conf.eth0.arp_filter": "1"
-      }
-    }
-    EOF
-
-    if [ $? -ne 0 ]; then
-      echo "Failed to create chained CNI config" >&2
-      exit 1
-    fi
-
-    echo "CNI chained setup completed successfully."
-    sleep infinity
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: cni-setup-daemonset
-  namespace: default
-  labels:
-    app: cni-setup
-spec:
-  selector:
-    matchLabels:
-      app: cni-setup
-  template:
-    metadata:
-      labels:
-        app: cni-setup
-    spec:
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-        - operator: Exists
-          effect: NoExecute
-      containers:
-      - name: setup
-        image: quay.io/fedora/fedora:40
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni-config
-          mountPath: /host/etc/cni/net.d
-        - name: script-volume
-          mountPath: /scripts
-        command: ["/bin/bash", "/scripts/setup.sh"]
-      volumes:
-      - name: cni-config
-        hostPath:
-          path: /etc/cni/net.d
-          type: Directory
-      - name: script-volume
-        configMap:
-          name: cni-setup-script
-          items:
-          - key: setup.sh
-            path: setup.sh

e2e/templates/subdirectory-chaining-pod.yml.j2

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: sysctl-modified
-spec:
-  containers:
-  - name: sysctl
-    image: quay.io/dosmith/fedora-procps
-    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
-    securityContext:
-      privileged: true

e2e/templates/subdirectory-chaining.yml.j2

@@ -1,95 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cni-setup-script
-  namespace: default
-data:
-  setup.sh: |
-    #!/bin/bash
-    set -euxo pipefail
-
-    DEFAULT_NETWORK_CNI_NAME="kindnet"
-
-    cleanup() {
-      echo "Cleaning up..."
-      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
-      if [ $? -ne 0 ]; then
-        echo "Failed to remove sysctltwiddle.conf" >&2
-        exit 1
-      fi
-      echo "Cleanup completed successfully"
-    }
-    trap cleanup EXIT
-
-    # Create the chained CNI directory if it doesn't exist
-    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
-    if [ $? -ne 0 ]; then
-      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
-      exit 1
-    fi
-
-    # Write the chained tuning CNI config
-    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
-    {
-      "cniVersion": "{{ CNI_VERSION }}",
-      "name": "sysctltwiddle",
-      "type": "tuning",
-      "sysctl": {
-        "net.ipv4.conf.IFNAME.arp_filter": "1"
-      }
-    }
-    EOF
-
-    if [ $? -ne 0 ]; then
-      echo "Failed to create chained CNI config" >&2
-      exit 1
-    fi
-
-    echo "CNI chained setup completed successfully."
-    sleep infinity
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: cni-setup-daemonset
-  namespace: default
-  labels:
-    app: cni-setup
-spec:
-  selector:
-    matchLabels:
-      app: cni-setup
-  template:
-    metadata:
-      labels:
-        app: cni-setup
-    spec:
-      hostNetwork: true
-      tolerations:
-        - operator: Exists
-          effect: NoSchedule
-        - operator: Exists
-          effect: NoExecute
-      containers:
-      - name: setup
-        image: quay.io/fedora/fedora:40
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni-config
-          mountPath: /host/etc/cni/net.d
-        - name: script-volume
-          mountPath: /scripts
-        command: ["/bin/bash", "/scripts/setup.sh"]
-      volumes:
-      - name: cni-config
-        hostPath:
-          path: /etc/cni/net.d
-          type: Directory
-      - name: script-volume
-        configMap:
-          name: cni-setup-script
-          items:
-          - key: setup.sh
-            path: setup.sh

e2e/test-default-route1.sh

@@ -3,14 +3,11 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f yamls/default-route1.yml
+kubectl create -f default-route1.yml
 kubectl wait --for=condition=ready -l app=default-route1 --timeout=300s pod
 
 echo "check default-route-worker1 interface: net1"
 kubectl exec default-route-worker1 -- ip a show dev net1
-if [ $? -ne 0 ];then
-       exit 1
-fi
 
 echo "check default-route-worker1 interface address: net1"
 ipaddr=$(kubectl exec default-route-worker1 -- ip -j a show  | jq -r \
@@ -28,9 +25,6 @@ fi
 
 echo "check default-route-worker2 interface: net1"
 kubectl exec default-route-worker2 -- ip a show dev net1
-if [ $? -ne 0 ];then
-       exit 1
-fi
 
 echo "check default-route-worker2 interface address: net1"
 ipaddr=$(kubectl exec default-route-worker2 -- ip -j a show  | jq -r \
@@ -47,4 +41,4 @@ if [ $ipaddr != "10.244.1.1" ]; then
 fi
 
 echo "cleanup resources"
-kubectl delete -f yamls/default-route1.yml
+kubectl delete -f default-route1.yml

e2e/test-dra-integration.sh

@@ -1,63 +0,0 @@
-#!/bin/sh
-set -o errexit
-
-export PATH=${PATH}:./bin
-
-# This test is using an example implementation of a DRA driver. This driver is mocking GPU resources. At our test we
-# don't care about what these resources are. We want to ensure that such resource is correctly passed in the Pod using
-# Multus configurations. A couple of notes:
-# - We explitictly pin the revision of the dra-example-driver to the branch `classic-dra` to indicate that the
-#   integration continues to work even when the dra-example-driver is updated. We know that classic-dra is supported
-#   in Kubernetes versions 1.26 to 1.30. Multus supports DRA in the aforementioned Kubernetes versions.
-# - The chart and latest is image is not published somewhere, therefore we have to build locally. This leads to slower
-#   e2e suite runs.
-echo "installing dra-example-driver"
-repo_path="repos/dra-example-driver"
-
-rm -rf $repo_path || true
-git clone --branch main https://github.com/kubernetes-sigs/dra-example-driver.git ${repo_path}
-MULTUS_DIR=$(pwd)
-cd ${repo_path}
-./demo/build-driver.sh
-KIND_CLUSTER_NAME=kind ./demo/scripts/load-driver-image-into-kind.sh
-cd "$MULTUS_DIR"
-chart_path=${repo_path}/deployments/helm/dra-example-driver/
-overriden_values_path=${chart_path}/overriden_values.yaml
-
-# With the thick plugin, in kind, the primary network on the control plane is not always working as expected. The pods
-# sometimes are not able to communicate with the control plane and the error looks like this:
-# failed to list *v1alpha2.PodSchedulingContext: Get "https://10.96.0.1:443/apis/resource.k8s.io/v1alpha2/podschedulingcontexts?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
-# We override the values here to schedule the controller on the worker nodes where the network is working as expected.
-cat <<EOF >> ${overriden_values_path}
-controller:
-  nodeSelector: null
-  tolerations: null
-EOF
-
-helm install \
-    -n dra-example-driver \
-    --create-namespace \
-    -f ${overriden_values_path} \
-    dra-example-driver \
-    ${chart_path}
-
-echo "installing testing pods"
-kubectl create -f yamls/dra-integration.yml
-kubectl wait --for=condition=ready -l app=dra-integration --timeout=300s pod
-
-echo "check dra-integration pod for DRA injected environment variable"
-
-# We can validate that the resource is correctly injected by checking an environment variable this dra driver is injecting
-# in the Pod.
-# https://github.com/kubernetes-sigs/dra-example-driver/blob/be2b8b1db47b8c757440e955ce5ced88c23bfe86/cmd/dra-example-kubeletplugin/cdi.go#L71C20-L71C44
-env_variable=$(kubectl exec dra-integration -- bash -c "echo \$DRA_RESOURCE_DRIVER_NAME | grep gpu.example.com")
-if [ $? -eq 0 ];then
-	echo "dra-integration pod has DRA injected environment variable"
-else
-	echo "dra-integration pod doesn't have DRA injected environment variable"
-	exit 1
-fi
-
-echo "cleanup resources"
-kubectl delete -f yamls/dra-integration.yml
-helm uninstall -n dra-example-driver dra-example-driver

e2e/test-simple-macvlan1.sh

@@ -3,17 +3,11 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f yamls/simple-macvlan1.yml
+kubectl create -f simple-macvlan1.yml
 kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
 
 echo "check macvlan1-worker1 interface: net1"
-net=$(kubectl exec macvlan1-worker1 -- ip a show dev net1)
-if [ $? -eq 0 ];then
-	echo "macvlan1-worker1 pod has net1 card"
-else
-	echo "macvlan1-worker1 pod has no net1 card"
-	exit 1
-fi
+kubectl exec macvlan1-worker1 -- ip a show dev net1
 
 echo "check macvlan1-worker1 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
@@ -23,13 +17,7 @@ if [ $ipaddr != "10.1.1.11" ]; then
 fi
 
 echo "check macvlan1-worker2 interface: net1"
-net2=$(kubectl exec macvlan1-worker2 -- ip a show dev net1)
-if [ $? -eq 0  ];then
-	echo "macvlan1-worker2 pod has net1 card"
-else
-	echo "macvlan1-worker2 pod has no net1 card"
-	exit 1
-fi
+kubectl exec macvlan1-worker2 -- ip a show dev net1
 
 echo "check macvlan1-worker2 interface address: net1"
 ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
@@ -39,4 +27,4 @@ if [ $ipaddr != "10.1.1.12" ]; then
 fi
 
 echo "cleanup resources"
-kubectl delete -f yamls/simple-macvlan1.yml
+kubectl delete -f simple-macvlan1.yml

e2e/test-simple-pod.sh

@@ -3,8 +3,8 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f yamls/simple-pod.yml
+kubectl create -f simple-pod.yml
 kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
 
 echo "cleanup resources"
-kubectl delete -f yamls/simple-pod.yml
+kubectl delete -f simple-pod.yml

e2e/test-subdirectory-chaining-passthru.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o nounset
-set -o pipefail
-
-export PATH=${PATH}:./bin
-
-TEST_POD_NAME="sysctl-modified"
-EXPECTED_BINARIES="${EXPECTED_BINARIES:-/opt/cni/bin/ptp /opt/cni/bin/portmap /opt/cni/bin/tuning}"
-EXPECTED_CNI_DIR="/etc/cni/net.d"
-
-# Reconfigure multus
-echo "Applying subdirectory chain passthru config..."
-kubectl apply -f yamls/subdirectory-chain-passthru-configupdate.yml
-
-# Restart the multus daemonset to pick up the new config
-echo "Restarting Multus DaemonSet..."
-kubectl rollout restart daemonset kube-multus-ds-amd64 -n kube-system
-kubectl rollout status daemonset/kube-multus-ds-amd64 -n kube-system
-
-# Debug: show CNI configs and binaries inside each Kind node
-echo "Checking CNI configs and binaries on nodes..."
-
-for node in $(kubectl get nodes --no-headers | awk '{print $1}'); do
-    container_name=$(docker ps --format '{{.Names}}' | grep "^${node}$")
-
-    echo "------"
-    echo "Node: ${node} (container: ${container_name})"
-    echo "Listing /opt/cni/bin contents..."
-    docker exec "${container_name}" ls -l /opt/cni/bin || echo "WARNING: /opt/cni/bin missing!"
-
-    echo "Checking expected binaries..."
-    for bin in $EXPECTED_BINARIES; do
-        echo "Checking for ${bin}..."
-        if docker exec "${container_name}" test -f "${bin}"; then
-            echo "SUCCESS: ${bin} found."
-        else
-            echo "FAIL: ${bin} NOT found!"
-        fi
-    done
-
-    echo "Listing /etc/cni/net.d configs..."
-    docker exec "${container_name}" ls -l ${EXPECTED_CNI_DIR} || echo "WARNING: ${EXPECTED_CNI_DIR} missing!"
-done
-echo "------"
-
-# Deploy the daemonset that will lay down the chained CNI config
-echo "Applying CNI setup DaemonSet..."
-kubectl apply -f yamls/subdirectory-chaining-passthru.yml
-
-# Wait for the daemonset pods to be ready (make sure they set up CNI config)
-echo "Waiting for CNI setup DaemonSet to be Ready..."
-kubectl rollout status daemonset/cni-setup-daemonset --timeout=300s
-
-# Deploy a test pod that will get chained CNI applied
-echo "Applying test pod..."
-kubectl apply -f yamls/subdirectory-chaining-pod.yml
-
-# Wait for the pod to be Ready
-echo "Waiting for test pod to be Ready..."
-kubectl wait --for=condition=ready pod/${TEST_POD_NAME} --timeout=300s
-
-# Check that the sysctl got set
-echo "Verifying sysctl arp_filter is set to 1 on eth0..."
-
-SYSCTL_VALUE=$(kubectl exec ${TEST_POD_NAME} -- sysctl -n net.ipv4.conf.eth0.arp_filter)
-
-if [ "$SYSCTL_VALUE" != "1" ]; then
-  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
-  exit 1
-else
-  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
-fi
-
-# Cleanup
-echo "Cleaning up test resources..."
-kubectl delete -f yamls/subdirectory-chaining-pod.yml
-kubectl delete -f yamls/subdirectory-chaining-passthru.yml
-
-echo "Test completed successfully."
-exit 0

e2e/test-subdirectory-chaining.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-set -o errexit
-
-export PATH=${PATH}:./bin
-
-TEST_POD_NAME="sysctl-modified"
-
-# Deploy the daemonset that will lay down the chained CNI config
-kubectl apply -f yamls/subdirectory-chaining.yml
-
-# Wait for the daemonset pods to be ready (we need the config to be laid down)
-kubectl rollout status daemonset/cni-setup-daemonset
-
-# Deploy a test pod that will get chained CNI applied
-kubectl apply -f yamls/subdirectory-chaining-pod.yml
-
-# Wait for the pod to be Ready
-kubectl wait --for=condition=ready pod/sysctl-modified --timeout=300s
-
-# Check that the sysctl got set properly inside the pod's eth0 interface
-echo "Verifying sysctl arp_filter is set to 1 on eth0"
-
-SYSCTL_VALUE=$(kubectl exec sysctl-modified -- sysctl -n net.ipv4.conf.eth0.arp_filter)
-
-if [ "$SYSCTL_VALUE" != "1" ]; then
-  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
-  exit 1
-else
-  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
-fi
-
-# 6. Clean up
-echo "Cleaning up test resources"
-kubectl delete -f yamls/subdirectory-chaining-pod.yml
-kubectl delete -f yamls/subdirectory-chaining.yml
-
-exit 0

examples/macvlan-pod.yml

@@ -54,4 +54,3 @@ spec:
     image: dougbtv/centos-network
     ports:
     - containerPort: 80
-  automountServiceAccountToken: false

examples/sriov-pod.yml

@@ -45,4 +45,3 @@ spec:
       limits:
         intel.com/sriov: '1'
   restartPolicy: "Never"
-  automountServiceAccountToken: false

go.mod

@@ -1,74 +1,53 @@
-module gopkg.in/k8snetworkplumbingwg/multus-cni.v4
+module gopkg.in/k8snetworkplumbingwg/multus-cni.v3
 
-go 1.23.4
+go 1.16
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/containernetworking/cni v1.3.0
-	github.com/containernetworking/plugins v1.7.1
-	github.com/fsnotify/fsnotify v1.9.0
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6
-	github.com/onsi/ginkgo/v2 v2.23.4
-	github.com/onsi/gomega v1.37.0
-	github.com/prometheus/client_golang v1.22.0
-	github.com/spf13/pflag v1.0.6
-	github.com/vishvananda/netlink v1.3.1
-	golang.org/x/net v0.41.0
-	golang.org/x/sys v0.33.0
-	google.golang.org/grpc v1.73.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
-	k8s.io/api v0.32.5
-	k8s.io/apimachinery v0.32.5
-	k8s.io/client-go v0.32.5
+	github.com/containernetworking/cni v0.8.1
+	github.com/containernetworking/plugins v0.9.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.3
+	github.com/pkg/errors v0.9.1
+	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	google.golang.org/grpc v1.27.1
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	k8s.io/api v0.20.10
+	k8s.io/apimachinery v0.20.10
+	k8s.io/client-go v0.20.10
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.130.1
-	k8s.io/kubelet v0.32.5
+	k8s.io/kubelet v0.0.0
+	k8s.io/kubernetes v1.20.10
 )
 
-require (
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
-	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/vishvananda/netns v0.0.5 // indirect
-	github.com/x448/float16 v0.8.4 // indirect
-	go.uber.org/automaxprocs v1.6.0 // indirect
-	golang.org/x/oauth2 v0.28.0 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
-	google.golang.org/protobuf v1.36.6 // indirect
-	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
-	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
-	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+replace (
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	k8s.io/api => k8s.io/api v0.20.10
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.10
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.10
+	k8s.io/apiserver => k8s.io/apiserver v0.20.10
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.10
+	k8s.io/client-go => k8s.io/client-go v0.20.10
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.10
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.10
+	k8s.io/code-generator => k8s.io/code-generator v0.20.10
+	k8s.io/component-base => k8s.io/component-base v0.20.10
+	k8s.io/component-helpers => k8s.io/component-helpers v0.20.10
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.10
+	k8s.io/cri-api => k8s.io/cri-api v0.20.10
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.10
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.10
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.10
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.10
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.10
+	k8s.io/kubectl => k8s.io/kubectl v0.20.10
+	k8s.io/kubelet => k8s.io/kubelet v0.20.10
+	k8s.io/kubernetes => k8s.io/kubernetes v1.20.10
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.10
+	k8s.io/metrics => k8s.io/metrics v0.20.10
+	k8s.io/mount-utils => k8s.io/mount-utils v0.20.10
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.10
 )

hack/build-go.sh

@@ -7,78 +7,54 @@ if [ ! -d ${DEST_DIR} ]; then
 	mkdir ${DEST_DIR}
 fi
 
-# Specify correspondingGOARCH from TARGETPLATFORM
-if [ "$TARGETPLATFORM" = "linux/amd64" ]; then
-	export GOARCH=amd64
-elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then
-	export GOARCH=arm64
-elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then
-	export GOARCH=arm
-elif [ "$TARGETPLATFORM" = "linux/ppc64le" ]; then
-	export GOARCH=ppc64le
-elif [ "$TARGETPLATFORM" = "linux/s390x" ]; then
-	export GOARCH=s390x
-fi
-
-# version information
-hasGit=true
-git version > /dev/null 2>&1 || hasGit=false
-GIT_SHA=""
-GIT_TREE_STATE=""
-GIT_TAG=""
-GIT_TAG_LAST=""
-RELEASE_STATUS=""
-if $hasGit; then
+# Add version/commit/date into binary
+# In case of TravisCI, need to check error code of 'git describe'.
+if [ -z "$VERSION" ]; then
 	set +e
-	GIT_SHA=$(git rev-parse --short HEAD)
-	# Tree state is "dirty" if there are uncommitted changes, untracked files are ignored
-	GIT_TREE_STATE=$(test -n "`git status --porcelain --untracked-files=no`" && echo "dirty" || echo "clean")
-	# Empty string if we are not building a tag
-	GIT_TAG=$(git describe --tags --abbrev=0 --exact-match 2>/dev/null || true)
-	# Find most recent tag
-	GIT_TAG_LAST=$(git describe --tags --abbrev=0 2>/dev/null || true)
+	git describe --tags --abbrev=0 > /dev/null 2>&1
+	if [ "$?" != "0" ]; then
+		VERSION="master"
+	else
+		VERSION=$(git describe --tags --abbrev=0)
+	fi
 	set -e
 fi
-
-# VERSION override mechanism if needed
-VERSION=${VERSION:-}
-if [[ -n "${VERSION}" || -n "${GIT_TAG}" ]]; then
-	RELEASE_STATUS=",released"
-fi
-
-if [ -z "$VERSION" ]; then
-	VERSION=$GIT_TAG_LAST
-fi
-# Add version/commit/date into binary
 DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
 COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
-LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version=${VERSION} \
-	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit=${COMMIT} \
-	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.gitTreeState=${GIT_TREE_STATE} \
-	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.releaseStatus=${RELEASE_STATUS} \
-	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date=${DATE}"
-export CGO_ENABLED=${CGO_ENABLED:-0}
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version=${VERSION:-master} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit=${COMMIT} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date=${DATE}"
+export CGO_ENABLED=0
 
-# build with go modules
-export GO111MODULE=on
+# this if... will be removed when gomodules goes default
+if [ "$GO111MODULE" == "off" ]; then
+	echo "Building plugin without go module"
+	echo "Warning: this will be deprecated in near future so please use go modules!"
 
-if [ -n "$MODMODE" ]; then
-	BUILD_ARGS=(-mod "$MODMODE")
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
+	REPO_PATH="${ORG_PATH}/multus-cni.v3"
+
+	if [ ! -h gopath/src/${REPO_PATH} ]; then
+		mkdir -p gopath/src/${ORG_PATH}
+		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
+	fi
+
+	export GO15VENDOREXPERIMENT=1
+	export GOBIN=${PWD}/bin
+	export GOPATH=${PWD}/gopath
+	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
+	go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/config-generation
+	go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/controller/
+else
+	# build with go modules
+	export GO111MODULE=on
+	BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+	if [ -n "$MODMODE" ]; then
+		BUILD_ARGS+=(-mod "$MODMODE")
+	fi
+
+	echo "Building plugins"
+	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
+	echo "Building spec generators"
+	go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/config-generation
+	echo "Building multus controller"
+	go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/controller/
 fi
-
-echo "Building multus"
-go build -o ${DEST_DIR}/multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
-echo "Building multus-daemon"
-go build -o "${DEST_DIR}"/multus-daemon ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-daemon
-echo "Building multus-shim"
-go build -o "${DEST_DIR}"/multus-shim ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-shim
-echo "Building install_multus"
-go build -o "${DEST_DIR}"/install_multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/install_multus
-echo "Building thin_entrypoint"
-go build -o "${DEST_DIR}"/thin_entrypoint ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
-echo "Building kubeconfig_generator"
-go build -o "${DEST_DIR}"/kubeconfig_generator ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/kubeconfig_generator
-echo "Building cert-approver"
-go build -o "${DEST_DIR}"/cert-approver ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/cert-approver
-echo "Building passthru CNI"
-go build -o "${DEST_DIR}"/passthru ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/passthru-cni

hack/test-go.sh

@@ -19,5 +19,5 @@ if [ "$GO111MODULE" == "off" ]; then
 	bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
 else
 	# test with go modules
-	bash -c "umask 0; go test -v -race -covermode=atomic -coverprofile=coverage.out ./..."
+	bash -c "umask 0; go test -v -covermode=count -coverprofile=coverage.out ./..."
 fi

images/Dockerfile

@@ -1,22 +1,17 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.23 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
-ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM gcr.io/distroless/base-debian12:latest
+FROM python:slim
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
 WORKDIR /
 
-COPY --from=build /usr/src/multus-cni/bin/install_multus /
-COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
-COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
-COPY --from=build /usr/src/multus-cni/bin/cert-approver /
-
-ENTRYPOINT ["/thin_entrypoint"]
+ADD ./images/entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm32

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "arm"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build arm container
+FROM arm32v7/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm64

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "arm64"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build arm64 container
+FROM arm64v8/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.debug

@@ -1,22 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.23 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ARG TARGETPLATFORM
-RUN  cd /usr/src/multus-cni && \
-     ./hack/build-go.sh
-
-FROM gcr.io/distroless/base-debian12:debug
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
-COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
-COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-WORKDIR /
-
-COPY --from=build /usr/src/multus-cni/bin/install_multus /
-COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
-COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
-COPY --from=build /usr/src/multus-cni/bin/cert-approver /
-
-ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.openshift

@@ -1,6 +1,5 @@
 # This dockerfile is specific to building Multus for OpenShift
-# The okd-builder image is locally built from https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile
-FROM local/okdbuilder:latest as builder
+FROM openshift/origin-release:golang-1.16 as builder
 
 ADD . /usr/src/multus-cni
 
@@ -8,16 +7,15 @@ WORKDIR /usr/src/multus-cni
 ENV GO111MODULE=off
 RUN ./hack/build-go.sh
 
-FROM quay.io/openshift/origin-base:latest
+FROM openshift/origin-base
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
 COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
-COPY --from=builder /usr/src/multus-cni/bin/install_multus /
-COPY --from=builder /usr/src/multus-cni/bin/thin_entrypoint /
+ADD ./images/entrypoint.sh /
 
 LABEL io.k8s.display-name="Multus CNI" \
       io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
       io.openshift.tags="openshift" \
       maintainer="Doug Smith <dosmith@redhat.com>"
 
-ENTRYPOINT ["/thin_entrypoint"]
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.ppc64le

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "ppc64le"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build ppc container
+FROM ppc64le/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.s390x

@@ -0,0 +1,21 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "s390x"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build s390x container
+FROM s390x/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.thick

@@ -1,18 +1,16 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.23 as build
+FROM golang:1.17.9 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
-ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 FROM debian:stable-slim
-LABEL org.opencontainers.image.source=https://github.com/k8snetworkplumbingwg/multus-cni
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
-COPY --from=build /usr/src/multus-cni/bin/cert-approver /
 WORKDIR /
 
 ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

images/entrypoint.sh

@@ -0,0 +1,475 @@
+#!/bin/bash
+
+# Always exit on errors.
+set -e
+
+# Trap sigterm
+function exitonsigterm() {
+  echo "Trapped sigterm, exiting."
+  exit 0
+}
+trap exitonsigterm SIGTERM
+
+# Set our known directories.
+CNI_CONF_DIR="/host/etc/cni/net.d"
+CNI_BIN_DIR="/host/opt/cni/bin"
+ADDITIONAL_BIN_DIR=""
+MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
+MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
+MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
+MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
+MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
+MULTUS_MASTER_CNI_FILE_NAME=""
+MULTUS_NAMESPACE_ISOLATION=false
+MULTUS_GLOBAL_NAMESPACES=""
+MULTUS_LOG_TO_STDERR=true
+MULTUS_LOG_LEVEL=""
+MULTUS_LOG_FILE=""
+MULTUS_READINESS_INDICATOR_FILE=""
+OVERRIDE_NETWORK_NAME=false
+MULTUS_CLEANUP_CONFIG_ON_EXIT=false
+RESTART_CRIO=false
+CRIO_RESTARTED_ONCE=false
+RENAME_SOURCE_CONFIG_FILE=false
+SKIP_BINARY_COPY=false
+
+# Give help text for parameters.
+function usage()
+{
+    echo -e "This is an entrypoint script for Multus CNI to overlay its configuration into"
+    echo -e "locations in a filesystem. The configuration file will be copied to the"
+    echo -e "corresponding configuration directory. When '--multus-conf-file=auto' is used,"
+    echo -e "00-multus.conf will be automatically generated from the CNI configuration file"
+    echo -e "of the master plugin (the first file in lexicographical order in cni-conf-dir)."
+    echo -e "When '--multus-master-cni-file-name' is used, 00-multus.conf will be"
+    echo -e "automatically generated from the specific file rather than the first file."
+    echo -e ""
+    echo -e "./entrypoint.sh"
+    echo -e "\t-h --help"
+    echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
+    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
+    echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
+    echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
+    echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
+    echo -e "\t--skip-multus-binary-copy=$SKIP_BINARY_COPY"
+    echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
+    echo -e "\t--multus-master-cni-file-name=$MULTUS_MASTER_CNI_FILE_NAME (empty by default, example: 10-calico.conflist)"
+    echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
+    echo -e "\t--global-namespaces=$MULTUS_GLOBAL_NAMESPACES (used only with --namespace-isolation=true)"
+    echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-to-stderr=$MULTUS_LOG_TO_STDERR (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--readiness-indicator-file=$MULTUS_READINESS_INDICATOR_FILE (used only with --multus-conf-file=auto)"
+    echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)"
+    echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)"
+}
+
+function log()
+{
+    echo "$(date --iso-8601=seconds) ${1}"
+}
+
+function error()
+{
+    log "ERR:  {$1}"
+}
+
+function warn()
+{
+    log "WARN: {$1}"
+}
+
+function checkCniVersion {
+      cniversion_python_tmpfile=$(mktemp)
+      cat << EOF > $cniversion_python_tmpfile
+import json, sys
+
+def version(v):
+    return [int(x) for x in v.split(".")]
+
+v_040 = version("0.4.0")
+v_top_level = sys.argv[2]
+with open(sys.argv[1], "r") as f:
+    v_nested = json.load(f)["cniVersion"]
+if version(v_top_level) >= v_040 and version(v_nested) < v_040:
+    msg = "Multus cni version is %s while master plugin cni version is %s"
+    print(msg % (v_top_level, v_nested))
+EOF
+      python3 $cniversion_python_tmpfile $1 $2
+}
+
+# Parse parameters given as arguments to this script.
+while [ "$1" != "" ]; do
+    PARAM=`echo $1 | awk -F= '{print $1}'`
+    VALUE=`echo $1 | awk -F= '{print $2}'`
+    case $PARAM in
+        -h | --help)
+            usage
+            exit
+            ;;
+        --cni-version)
+            CNI_VERSION=$VALUE
+            ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
+        --cni-conf-dir)
+            CNI_CONF_DIR=$VALUE
+            ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
+        --multus-conf-file)
+            MULTUS_CONF_FILE=$VALUE
+            ;;
+        --multus-kubeconfig-file-host)
+            MULTUS_KUBECONFIG_FILE_HOST=$VALUE
+            ;;
+        --multus-master-cni-file-name)
+            MULTUS_MASTER_CNI_FILE_NAME=$VALUE
+            ;;
+        --namespace-isolation)
+            MULTUS_NAMESPACE_ISOLATION=$VALUE
+            ;;
+        --global-namespaces)
+            MULTUS_GLOBAL_NAMESPACES=$VALUE
+            ;;
+        --multus-log-to-stderr)
+            MULTUS_LOG_TO_STDERR=$VALUE
+            ;;
+        --multus-log-level)
+            MULTUS_LOG_LEVEL=$VALUE
+            ;;
+        --multus-log-file)
+            MULTUS_LOG_FILE=$VALUE
+            ;;
+        --multus-autoconfig-dir)
+            MULTUS_AUTOCONF_DIR=$VALUE
+            ;;
+        --override-network-name)
+            OVERRIDE_NETWORK_NAME=$VALUE
+            ;;
+        --cleanup-config-on-exit)
+            MULTUS_CLEANUP_CONFIG_ON_EXIT=$VALUE
+            ;;
+        --restart-crio)
+            RESTART_CRIO=$VALUE
+            ;;
+        --rename-conf-file)
+            RENAME_SOURCE_CONFIG_FILE=$VALUE
+            ;;
+        --additional-bin-dir)
+            ADDITIONAL_BIN_DIR=$VALUE
+            ;;
+        --skip-multus-binary-copy)
+            SKIP_BINARY_COPY=$VALUE
+            ;;
+        --readiness-indicator-file)
+            MULTUS_READINESS_INDICATOR_FILE=$VALUE
+            ;;
+        *)
+            warn "unknown parameter \"$PARAM\""
+            ;;
+    esac
+    shift
+done
+
+
+# Create array of known locations
+declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE)
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  arr+=($MULTUS_CONF_FILE)
+fi
+
+
+# Loop through and verify each location each.
+for i in "${arr[@]}"
+do
+  if [ ! -e "$i" ]; then
+    warn "Location $i does not exist"
+    exit 1;
+  fi
+done
+
+# Copy files into place and atomically move into final binary name
+if [ "$SKIP_BINARY_COPY" = false ]; then
+  cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR/_multus
+  mv -f $CNI_BIN_DIR/_multus $CNI_BIN_DIR/multus
+else
+  log "Entrypoint skipped copying Multus binary."
+fi
+
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
+fi
+
+# Make a multus.d directory (for our kubeconfig)
+
+mkdir -p $CNI_CONF_DIR/multus.d
+MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
+
+# ------------------------------- Generate a "kube-config"
+# Inspired by: https://tinyurl.com/y7r2knme
+SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
+SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
+
+
+# Check if we're running as a k8s pod.
+if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
+  # We're running as a k8d pod - expect some variables.
+  if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
+    error "KUBERNETES_SERVICE_HOST not set"; exit 1;
+  fi
+  if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
+    error "KUBERNETES_SERVICE_PORT not set"; exit 1;
+  fi
+
+  if [ "$SKIP_TLS_VERIFY" == "true" ]; then
+    TLS_CFG="insecure-skip-tls-verify: true"
+  elif [ -f "$KUBE_CA_FILE" ]; then
+    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+  fi
+
+  # Write a kubeconfig file for the CNI plugin.  Do this
+  # to skip TLS verification for now.  We should eventually support
+  # writing more complete kubeconfig files. This is only used
+  # if the provided CNI network config references it.
+  touch $MULTUS_TEMP_KUBECONFIG
+  chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
+  # Write the kubeconfig to a temp file first.
+  cat > $MULTUS_TEMP_KUBECONFIG <<EOF
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT}
+    $TLS_CFG
+users:
+- name: multus
+  user:
+    token: "${SERVICEACCOUNT_TOKEN}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+EOF
+
+  # Atomically move the temp kubeconfig to its permanent home.
+  mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG
+
+else
+  warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
+fi
+
+# ---------------------- end Generate a "kube-config".
+
+# ------------------------------- Generate "00-multus.conf"
+
+function generateMultusConf {
+if [ "$MULTUS_CONF_FILE" == "auto" ]; then
+  log "Generating Multus configuration file using files in $MULTUS_AUTOCONF_DIR..."
+  found_master=false
+  tries=0
+  while [ $found_master == false ]; do
+    if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then
+        MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME"
+    else
+        MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
+    fi
+    if [ "$MASTER_PLUGIN" == "" ]; then
+      if [ $tries -lt 600 ]; then
+        if ! (($tries % 5)); then
+          log "Attempting to find master plugin configuration, attempt $tries"
+        fi
+        let "tries+=1"
+        sleep 1;
+      else
+        error "Multus could not be configured: no master plugin was found."
+        exit 1;
+      fi
+    else
+      log "Using MASTER_PLUGIN: $MASTER_PLUGIN"
+
+      found_master=true
+
+      ISOLATION_STRING=""
+      if [ "$MULTUS_NAMESPACE_ISOLATION" == true ]; then
+        ISOLATION_STRING="\"namespaceIsolation\": true,"
+      fi
+
+      GLOBAL_NAMESPACES_STRING=""
+      if [ ! -z "${MULTUS_GLOBAL_NAMESPACES// }" ]; then
+        GLOBAL_NAMESPACES_STRING="\"globalNamespaces\": \"$MULTUS_GLOBAL_NAMESPACES\","
+      fi
+
+      LOG_TO_STDERR_STRING=""
+      if [ "$MULTUS_LOG_TO_STDERR" == false ]; then
+        LOG_TO_STDERR_STRING="\"logToStderr\": false,"
+      fi
+
+
+      LOG_LEVEL_STRING=""
+      if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
+        case "$MULTUS_LOG_LEVEL" in
+          debug)
+              ;;
+          error)
+              ;;
+          panic)
+              ;;
+          verbose)
+              ;;
+          *)
+              error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
+              usage
+              exit 1
+        esac
+        LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
+      fi
+
+      LOG_FILE_STRING=""
+      if [ ! -z "${MULTUS_LOG_FILE// }" ]; then
+        LOG_FILE_STRING="\"logFile\": \"$MULTUS_LOG_FILE\","
+      fi
+
+      CNI_VERSION_STRING=""
+      if [ ! -z "${CNI_VERSION// }" ]; then
+        CNI_VERSION_STRING="\"cniVersion\": \"$CNI_VERSION\","
+      fi
+
+      ADDITIONAL_BIN_DIR_STRING=""
+      if [ ! -z "${ADDITIONAL_BIN_DIR// }" ]; then
+        ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\","
+      fi
+
+
+      READINESS_INDICATOR_FILE_STRING=""
+      if [ ! -z "${MULTUS_READINESS_INDICATOR_FILE// }" ]; then
+        READINESS_INDICATOR_FILE_STRING="\"readinessindicatorfile\": \"$MULTUS_READINESS_INDICATOR_FILE\","
+      fi
+
+      if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
+        MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
+            python3 -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
+      else
+        MASTER_PLUGIN_NET_NAME="multus-cni-network"
+      fi
+
+      capabilities_python_filter_tmpfile=$(mktemp)
+      cat << EOF > $capabilities_python_filter_tmpfile
+import json,sys
+conf = json.load(sys.stdin)
+capabilities = {}
+if 'plugins' in conf:
+    for capa in [p['capabilities'] for p in conf['plugins'] if 'capabilities' in p]:
+        capabilities.update({capability:enabled for (capability,enabled) in capa.items() if enabled})
+elif 'capabilities' in conf:
+    capabilities.update({capability:enabled for (capability,enabled) in conf['capabilities'] if enabled})
+if len(capabilities) > 0:
+    print("""\"capabilities\": """ + json.dumps(capabilities) + ",")
+else:
+    print("")
+EOF
+
+      NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
+            python3 $capabilities_python_filter_tmpfile)"
+      rm $capabilities_python_filter_tmpfile
+      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING"
+
+      MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
+      MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
+      log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
+      CHECK_CNI_VERSION=$(checkCniVersion $MASTER_PLUGIN_LOCATION $CNI_VERSION)
+      if [ "$CHECK_CNI_VERSION" != "" ] ; then
+        error "$CHECK_CNI_VERSION"
+        exit 1
+      fi
+
+      CONF=$(cat <<-EOF
+        {
+          $CNI_VERSION_STRING
+          "name": "$MASTER_PLUGIN_NET_NAME",
+          "type": "multus",
+          $NESTED_CAPABILITIES_STRING
+          $ISOLATION_STRING
+          $GLOBAL_NAMESPACES_STRING
+          $LOG_TO_STDERR_STRING
+          $LOG_LEVEL_STRING
+          $LOG_FILE_STRING
+          $ADDITIONAL_BIN_DIR_STRING
+          $READINESS_INDICATOR_FILE_STRING
+          "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
+          "delegates": [
+            $MASTER_PLUGIN_JSON
+          ]
+        }
+EOF
+      )
+      tmpfile=$(mktemp)
+      echo $CONF > $tmpfile
+      mv $tmpfile $CNI_CONF_DIR/00-multus.conf
+      log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
+      echo $CONF
+
+      # If we're not performing the cleanup on exit, we can safely rename the config file.
+      if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
+        mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old
+        log "Original master file moved to ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old"
+      fi
+
+      if [ "$RESTART_CRIO" == true ]; then
+        # Restart CRIO only once.
+        if [ "$CRIO_RESTARTED_ONCE" == false ]; then
+          log "Restarting crio"
+          systemctl restart crio
+          CRIO_RESTARTED_ONCE=true
+        fi
+      fi
+    fi
+  done
+fi
+}
+generateMultusConf
+
+# ---------------------- end Generate "00-multus.conf".
+
+# Enter either sleep loop, or watch loop...
+if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
+  log "Entering watch loop..."
+  while true; do
+    # Check and see if the original master plugin configuration exists...
+    if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then
+      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Allowing 45 seconds for its restoration..."
+      sleep 10
+      for i in {1..35}
+      do
+        if [ -f "$MASTER_PLUGIN_LOCATION" ]; then
+          log "Master plugin @ $MASTER_PLUGIN_LOCATION was restored. Regenerating given configuration."
+          break
+        fi
+        sleep 1
+      done
+
+      generateMultusConf
+      log "Continuing watch loop after configuration regeneration..."
+    fi
+    sleep 1
+  done
+else
+  log "Entering sleep (success)..."
+  if tty -s; then
+	  read
+  else
+	  sleep infinity
+  fi
+fi

pkg/checkpoint/checkpoint.go

@@ -1,5 +1,4 @@
 // Copyright (c) 2018 Intel Corporation
-// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,15 +11,16 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+//
 
 package checkpoint
 
 import (
 	"encoding/json"
-	"os"
+	"io/ioutil"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -33,7 +33,7 @@ type PodDevicesEntry struct {
 	PodUID        string
 	ContainerName string
 	ResourceName  string
-	DeviceIDs     map[int64][]string
+	DeviceIDs     []string
 	AllocResp     []byte
 }
 
@@ -72,7 +72,7 @@ func getCheckpoint(filePath string) (types.ResourceClient, error) {
 func (cp *checkpoint) getPodEntries() error {
 
 	cpd := &checkpointFileData{}
-	rawBytes, err := os.ReadFile(cp.fileName)
+	rawBytes, err := ioutil.ReadFile(cp.fileName)
 	if err != nil {
 		return logging.Errorf("getPodEntries: error reading file %s\n%v\n", checkPointfile, err)
 	}
@@ -86,7 +86,7 @@ func (cp *checkpoint) getPodEntries() error {
 	return nil
 }
 
-// GetPodResourceMap returns an instance of a map of ResourceInfo
+// GetComputeDeviceMap returns an instance of a map of ResourceInfo
 func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
 	podID := string(pod.UID)
 	resourceMap := make(map[string]*types.ResourceInfo)
@@ -97,14 +97,12 @@ func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resource
 	for _, pod := range cp.podEntires {
 		if pod.PodUID == podID {
 			entry, ok := resourceMap[pod.ResourceName]
-			if !ok {
-				// new entry
-				entry = &types.ResourceInfo{}
-				resourceMap[pod.ResourceName] = entry
-			}
-			for _, v := range pod.DeviceIDs {
+			if ok {
 				// already exists; append to it
-				entry.DeviceIDs = append(entry.DeviceIDs, v...)
+				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
+			} else {
+				// new entry
+				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
 			}
 		}
 	}

pkg/checkpoint/checkpoint_test.go

@@ -1,32 +1,16 @@
-// Copyright (c) 2018 Intel Corporation
-// Copyright (c) 2021 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package checkpoint
 
-// disable dot-imports only for testing
-//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
 
-	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"io/ioutil"
 	"testing"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
@@ -41,7 +25,7 @@ type fakeCheckpoint struct {
 }
 
 func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error {
-	return os.WriteFile(fc.fileName, inBytes, 0600)
+	return ioutil.WriteFile(fc.fileName, inBytes, 0600)
 }
 
 func (fc *fakeCheckpoint) DeleteFile() error {
@@ -61,11 +45,10 @@ var _ = BeforeSuite(func() {
 				"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
 				"ContainerName": "appcntr1",
 				"ResourceName": "intel.com/sriov_net_A",
-				"DeviceIDs": {"-1": [
-					"0000:03:02.3",
-					"0000:03:02.0"
-					]
-				},
+				"DeviceIDs": [
+				"0000:03:02.3",
+				"0000:03:02.0"
+				],
 				"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
 			}
 			],
@@ -160,10 +143,10 @@ var _ = Describe("Kubelet checkpoint data read operations", func() {
 						"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
 						"ContainerName": "appcntr1",
 						"ResourceName": "intel.com/sriov_net_A",
-						"DeviceIDs": { "-1": [
+						"DeviceIDs": [
 						"0000:03:02.3",
 						"0000:03:02.0"
-						] },
+						],
 						"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
 					}
 					],

pkg/checkpoint/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2022 Multus Authors
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/cmdutils/cmdutils_suite_test.go

@@ -1,30 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package cmdutils is the package that contains utilities for multus command
-package cmdutils
-
-// disable dot-imports only for testing
-//revive:disable:dot-imports
-import (
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-
-	"testing"
-)
-
-func TestServer(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "cmdutils")
-}

pkg/cmdutils/utils.go

@@ -1,84 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package cmdutils is the package that contains utilities for multus command
-package cmdutils
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-)
-
-// CopyFileAtomic does file copy atomically
-func CopyFileAtomic(srcFilePath, destDir, tempFileName, destFileName string) error {
-	tempFilePath := filepath.Join(destDir, tempFileName)
-	// check temp filepath and remove old file if exists
-	if _, err := os.Stat(tempFilePath); err == nil {
-		err = os.Remove(tempFilePath)
-		if err != nil {
-			return fmt.Errorf("cannot remove old temp file %q: %v", tempFilePath, err)
-		}
-	}
-
-	// create temp file
-	f, err := os.CreateTemp(destDir, tempFileName)
-	defer f.Close()
-	if err != nil {
-		return fmt.Errorf("cannot create temp file %q in %q: %v", tempFileName, destDir, err)
-	}
-
-	srcFile, err := os.Open(srcFilePath)
-	if err != nil {
-		return fmt.Errorf("cannot open file %q: %v", srcFilePath, err)
-	}
-	defer srcFile.Close()
-
-	// Copy file to tempfile
-	_, err = io.Copy(f, srcFile)
-	if err != nil {
-		f.Close()
-		os.Remove(tempFilePath)
-		return fmt.Errorf("cannot write data to temp file %q: %v", tempFilePath, err)
-	}
-	if err := f.Sync(); err != nil {
-		return fmt.Errorf("cannot flush temp file %q: %v", tempFilePath, err)
-	}
-	if err := f.Close(); err != nil {
-		return fmt.Errorf("cannot close temp file %q: %v", tempFilePath, err)
-	}
-
-	// change file mode if different
-	destFilePath := filepath.Join(destDir, destFileName)
-	_, err = os.Stat(destFilePath)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	srcFileStat, err := os.Stat(srcFilePath)
-	if err != nil {
-		return err
-	}
-
-	if err := os.Chmod(f.Name(), srcFileStat.Mode()); err != nil {
-		return fmt.Errorf("cannot set stat on temp file %q: %v", f.Name(), err)
-	}
-
-	// replace file with tempfile
-	if err := os.Rename(f.Name(), destFilePath); err != nil {
-		return fmt.Errorf("cannot replace %q with temp file %q: %v", destFilePath, tempFilePath, err)
-	}
-
-	return nil
-}

pkg/cmdutils/utils_test.go

@@ -1,72 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package cmdutils is the package that contains utilities for multus command
-package cmdutils
-
-// disable dot-imports only for testing
-//revive:disable:dot-imports
-import (
-	"fmt"
-	"os"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("thin entrypoint testing", func() {
-	It("Run CopyFileAtomic()", func() {
-		// create directory and files
-		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
-		Expect(err).NotTo(HaveOccurred())
-
-		// create source directory
-		srcDir := fmt.Sprintf("%s/src", tmpDir)
-		err = os.Mkdir(srcDir, 0755)
-		Expect(err).NotTo(HaveOccurred())
-
-		// create destination directory
-		destDir := fmt.Sprintf("%s/dest", tmpDir)
-		err = os.Mkdir(destDir, 0755)
-		Expect(err).NotTo(HaveOccurred())
-
-		// sample source file
-		srcFilePath := fmt.Sprintf("%s/sampleInput", srcDir)
-		err = os.WriteFile(srcFilePath, []byte("sampleInputABC"), 0744)
-		Expect(err).NotTo(HaveOccurred())
-
-		// old files in dest
-		destFileName := "sampleInputDest"
-		destFilePath := fmt.Sprintf("%s/%s", destDir, destFileName)
-		err = os.WriteFile(destFilePath, []byte("inputOldXYZ"), 0611)
-		Expect(err).NotTo(HaveOccurred())
-
-		tempFileName := "temp_file"
-		err = CopyFileAtomic(srcFilePath, destDir, tempFileName, destFileName)
-		Expect(err).NotTo(HaveOccurred())
-
-		// check file mode
-		stat, err := os.Stat(destFilePath)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(stat.Mode()).To(Equal(os.FileMode(0744)))
-
-		// check file contents
-		destFileByte, err := os.ReadFile(destFilePath)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(destFileByte).To(Equal([]byte("sampleInputABC")))
-
-		err = os.RemoveAll(tmpDir)
-		Expect(err).NotTo(HaveOccurred())
-	})
-})

pkg/config/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package config is the package that contains multus cni config related
+// utilities.
+package config

pkg/config/generator.go

@@ -0,0 +1,333 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+)
+
+const (
+	configListCapabilityKey   = "plugins"
+	singleConfigCapabilityKey = "capabilities"
+)
+
+// LogOptionFunc mutates the `LoggingOptions` object
+type LogOptionFunc func(logOptions *LogOptions)
+
+// Option mutates the `conf` object
+type Option func(conf *MultusConf)
+
+// MultusConf holds the multus configuration, and persists it to disk
+type MultusConf struct {
+	BinDir                   string          `json:"binDir,omitempty"`
+	Capabilities             map[string]bool `json:"capabilities,omitempty"`
+	CNIVersion               string          `json:"cniVersion"`
+	Delegates                []interface{}   `json:"delegates"`
+	LogFile                  string          `json:"logFile,omitempty"`
+	LogLevel                 string          `json:"logLevel,omitempty"`
+	LogToStderr              bool            `json:"logToStderr,omitempty"`
+	LogOptions               *LogOptions     `json:"logOptions,omitempty"`
+	Kubeconfig               string          `json:"kubeconfig"`
+	Name                     string          `json:"name"`
+	NamespaceIsolation       bool            `json:"namespaceIsolation,omitempty"`
+	RawNonIsolatedNamespaces string          `json:"globalNamespaces,omitempty"`
+	ReadinessIndicatorFile   string          `json:"readinessindicatorfile,omitempty"`
+	Type                     string          `json:"type"`
+}
+
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// NewMultusConfig creates a basic configuration generator. It can be mutated
+// via the `With...` methods.
+func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
+	multusConfig := &MultusConf{
+		Name:         MultusDefaultNetworkName,
+		CNIVersion:   cniVersion,
+		Type:         pluginName,
+		Capabilities: map[string]bool{},
+		Kubeconfig:   kubeconfig,
+		Delegates:    []interface{}{},
+	}
+
+	err := multusConfig.Mutate(configurationOptions...)
+	return multusConfig, err
+}
+
+// CheckVersionCompatibility checks compatibilty of the
+// top level cni version with the delegate cni version.
+// Since version 0.4.0, CHECK was introduced, which
+// causes incompatibility.
+func CheckVersionCompatibility(mc *MultusConf) error {
+	const versionFmt = "delegate cni version is %s while top level cni version is %s"
+	v040, _ := semver.Make("0.4.0")
+	multusCNIVersion, err := semver.Make(mc.CNIVersion)
+
+	if err != nil {
+		return errors.New("couldn't get top level cni version")
+	}
+
+	if multusCNIVersion.GTE(v040) {
+		for _, delegate := range mc.Delegates {
+			delegatesMap, ok := delegate.(map[string]interface{})
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			delegateVersion, ok := delegatesMap["cniVersion"].(string)
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			v, err := semver.Make(delegateVersion)
+			if err != nil {
+				return err
+			}
+			if v.LT(v040) {
+				return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Generate generates the multus configuration from whatever state is currently
+// held
+func (mc *MultusConf) Generate() (string, error) {
+	data, err := json.Marshal(mc)
+	return string(data), err
+}
+
+// Mutate updates the MultusConf attributes according to the provided
+// configuration `Option`s
+func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
+	for _, configOption := range configurationOptions {
+		configOption(mc)
+	}
+
+	return CheckVersionCompatibility(mc)
+}
+
+// WithNamespaceIsolation mutates the inner state to enable the
+// NamespaceIsolation attribute
+func WithNamespaceIsolation() Option {
+	return func(conf *MultusConf) {
+		conf.NamespaceIsolation = true
+	}
+}
+
+// WithGlobalNamespaces mutates the inner state to set the
+// RawNonIsolatedNamespaces attribute
+func WithGlobalNamespaces(globalNamespaces string) Option {
+	return func(conf *MultusConf) {
+		conf.RawNonIsolatedNamespaces = globalNamespaces
+	}
+}
+
+// WithLogToStdErr mutates the inner state to enable the
+// WithLogToStdErr attribute
+func WithLogToStdErr() Option {
+	return func(conf *MultusConf) {
+		conf.LogToStderr = true
+	}
+}
+
+// WithLogLevel mutates the inner state to set the
+// LogLevel attribute
+func WithLogLevel(logLevel string) Option {
+	return func(conf *MultusConf) {
+		conf.LogLevel = logLevel
+	}
+}
+
+// WithLogFile mutates the inner state to set the
+// logFile attribute
+func WithLogFile(logFile string) Option {
+	return func(conf *MultusConf) {
+		conf.LogFile = logFile
+	}
+}
+
+// WithLogOptions mutates the inner state to set the
+// LogOptions attribute
+func WithLogOptions(logOptions *LogOptions) Option {
+	return func(conf *MultusConf) {
+		conf.LogOptions = logOptions
+	}
+}
+
+// WithReadinessFileIndicator mutates the inner state to set the
+// ReadinessIndicatorFile attribute
+func WithReadinessFileIndicator(path string) Option {
+	return func(conf *MultusConf) {
+		conf.ReadinessIndicatorFile = path
+	}
+}
+
+// WithAdditionalBinaryFileDir mutates the inner state to set the
+// BinDir attribute
+func WithAdditionalBinaryFileDir(directoryPath string) Option {
+	return func(conf *MultusConf) {
+		conf.BinDir = directoryPath
+	}
+}
+
+// WithOverriddenName mutates the inner state to set the
+// Name attribute
+func WithOverriddenName(networkName string) Option {
+	return func(conf *MultusConf) {
+		conf.Name = networkName
+	}
+}
+
+func withCapabilities(cniData interface{}) Option {
+	var enabledCapabilities []string
+	var pluginsList []interface{}
+	cniDataMap, ok := cniData.(map[string]interface{})
+	if ok {
+		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
+			pluginsList = pluginsListEntry.([]interface{})
+		}
+	}
+
+	if len(pluginsList) > 0 {
+		for _, pluginData := range pluginsList {
+			enabledCapabilities = append(
+				enabledCapabilities,
+				extractCapabilities(pluginData)...)
+		}
+	} else {
+		enabledCapabilities = extractCapabilities(cniData)
+	}
+
+	return func(conf *MultusConf) {
+		for _, capability := range enabledCapabilities {
+			conf.Capabilities[capability] = true
+		}
+	}
+}
+
+func withDelegates(primaryCNIConfigData map[string]interface{}) Option {
+	return func(conf *MultusConf) {
+		conf.Delegates = []interface{}{primaryCNIConfigData}
+	}
+}
+
+// MutateLogOptions update the LoggingOptions of the MultusConf according
+// to the provided configuration `loggingOptions`
+func MutateLogOptions(logOption *LogOptions, logOptionFunc ...LogOptionFunc) {
+	for _, loggingOption := range logOptionFunc {
+		loggingOption(logOption)
+	}
+}
+
+// WithLogMaxSize mutates the inner state to set the
+// logMaxSize attribute
+func WithLogMaxSize(maxSize *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxSize = maxSize
+	}
+}
+
+// WithLogMaxAge mutates the inner state to set the
+// logMaxAge attribute
+func WithLogMaxAge(maxAge *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxAge = maxAge
+	}
+}
+
+// WithLogMaxBackups mutates the inner state to set the
+// logMaxBackups attribute
+func WithLogMaxBackups(maxBackups *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxBackups = maxBackups
+	}
+}
+
+// WithLogCompress mutates the inner state to set the
+// logCompress attribute
+func WithLogCompress(compress *bool) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.Compress = compress
+	}
+}
+
+func extractCapabilities(capabilitiesInterface interface{}) []string {
+	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
+	if !ok {
+		return nil
+	}
+	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	var enabledCapabilities []string
+	if len(capabilities) > 0 {
+		for capName, isCapabilityEnabled := range capabilities {
+			if isCapabilityEnabled.(bool) {
+				enabledCapabilities = append(enabledCapabilities, capName)
+			}
+		}
+	}
+	return enabledCapabilities
+}
+
+func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
+	if remainingTries == 0 {
+		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
+	}
+	var cniPluginConfigs []string
+	files, err := ioutil.ReadDir(cniConfigDirPath)
+	if err != nil {
+		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
+	}
+
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "00-multus") {
+			continue
+		}
+		fileExtension := filepath.Ext(file.Name())
+		if fileExtension == ".conf" || fileExtension == ".conflist" {
+			cniPluginConfigs = append(cniPluginConfigs, file.Name())
+		}
+	}
+
+	if len(cniPluginConfigs) == 0 {
+		time.Sleep(time.Second)
+		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
+	}
+	sort.Strings(cniPluginConfigs)
+	return cniPluginConfigs[0], nil
+}

pkg/config/generator_test.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"testing"
+)
+
+const (
+	primaryCNIName = "myCNI"
+	cniVersion     = "0.4.0"
+	kubeconfig     = "/a/b/c/kubeconfig.kubeconfig"
+)
+
+type testCase struct {
+	t                        *testing.T
+	configGenerationFunction func() (string, error)
+}
+
+var primaryCNIConfig = map[string]interface{}{
+	"cniVersion":         "1.0.0",
+	"name":               "ovn-kubernetes",
+	"type":               "ovn-k8s-cni-overlay",
+	"ipam":               "{}",
+	"dns":                "{}",
+	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+	"logLevel":           "5",
+	"logfile-maxsize":    100,
+	"logfile-maxbackups": 5,
+	"logfile-maxage":     5,
+}
+
+func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
+	multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
+	if err != nil {
+		return multusConfig, err
+	}
+	return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{})))
+}
+
+func TestBasicMultusConfig(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig)
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithNamespaceIsolation())
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithReadinessIndicator(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithReadinessFileIndicator("/a/b/u/it-lives"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogLevel("notice"),
+		WithLogToStdErr(),
+		WithLogFile("/u/y/w/log.1"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLogOptionsConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(&LogOptions{
+			MaxAge:     testutils.Int(5),
+			MaxSize:    testutils.Int(100),
+			MaxBackups: testutils.Int(5),
+			Compress:   testutils.Bool(true),
+		}))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5,\"maxSize\":100,\"maxBackups\":5,\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxAge(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxAge(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxSize(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxSize(testutils.Int(100)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxSize\":100},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogBackups(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxBackups(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxBackups\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogCompress(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogCompress(testutils.Bool(true)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithGlobalNamespace(t *testing.T) {
+	const globalNamespace = "come-along-ns"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithGlobalNamespaces(globalNamespace))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
+	const anotherCNIBinDir = "a-dir-somewhere"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithAdditionalBinaryFileDir(anotherCNIBinDir))
+	assertError(t, err, nil)
+	expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`
+{
+    "plugins": [
+        {
+            "capabilities": {
+                "portMappings": true
+            }
+        },
+        {
+            "capabilities": {
+                "tuning": false
+            }
+        }
+    ]
+}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func assertError(t *testing.T, actual error, expected error) {
+	if actual != nil && expected != nil {
+		if actual.Error() != expected.Error() {
+			t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
+		}
+	}
+
+	if actual == nil && expected != nil {
+		t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
+	} else if actual != nil && expected == nil {
+		t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
+	}
+}
+
+func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
+	return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
+}
+
+func TestVersionIncompatibility(t *testing.T) {
+	const delegateCNIVersion = "0.3.0"
+
+	primaryCNIConfigOld := primaryCNIConfig
+	tmpVer := primaryCNIConfig["cniVersion"]
+	primaryCNIConfig["cniVersion"] = delegateCNIVersion
+	_, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfigOld)
+	primaryCNIConfig["cniVersion"] = tmpVer
+
+	assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
+}
+
+func TestMultusConfigWithOverriddenName(t *testing.T) {
+	newNetworkName := "mega-net-2000"
+	multusConfig, _ := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithOverriddenName(newNetworkName))
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
+	return &testCase{
+		t:                        t,
+		configGenerationFunction: configGenerationFunc,
+	}
+}
+
+func (tc testCase) assertResult(expectedResult string) {
+	multusCNIConfig, err := tc.configGenerationFunction()
+	if err != nil {
+		tc.t.Fatalf("error generating multus configuration: %v", err)
+	}
+	if multusCNIConfig != expectedResult {
+		tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
+	}
+}
+
+func documentHelper(pluginInfo string) interface{} {
+	dp, _ := documentCNIData([]byte(pluginInfo))
+	return dp
+}
+
+func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/fsnotify/fsnotify"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+)
+
+// MultusDefaultNetworkName holds the default name of the multus network
+const (
+	multusConfigFileName     = "00-multus.conf"
+	MultusDefaultNetworkName = "multus-cni-network"
+	userRWPermission         = 0600
+)
+
+// Manager monitors the configuration of the primary CNI plugin, and
+// regenerates multus configuration whenever it gets updated.
+type Manager struct {
+	cniConfigData        map[string]interface{}
+	configWatcher        *fsnotify.Watcher
+	multusConfig         *MultusConf
+	multusConfigDir      string
+	multusConfigFilePath string
+	primaryCNIConfigPath string
+}
+
+// NewManager returns a config manager object, configured to persist the
+// configuration to `multusAutoconfigDir`. This constructor will auto-discover
+// the primary CNI for which it will delegate.
+func NewManager(config MultusConf, multusAutoconfigDir string) (*Manager, error) {
+	defaultCNIPluginName, err := primaryCNIPluginName(multusAutoconfigDir)
+	if err != nil {
+		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+		return nil, err
+	}
+	return newManager(config, multusAutoconfigDir, defaultCNIPluginName)
+}
+
+// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
+// configured to persist the configuration to `multusAutoconfigDir`. This
+// constructor will use the primary CNI plugin indicated by the user, via the
+// primaryCNIPluginName variable.
+func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir string, primaryCNIPluginName string) (*Manager, error) {
+	return newManager(config, multusAutoconfigDir, primaryCNIPluginName)
+}
+
+func newManager(config MultusConf, multusConfigDir string, defaultCNIPluginName string) (*Manager, error) {
+	watcher, err := newWatcher(multusConfigDir)
+	if err != nil {
+		return nil, err
+	}
+
+	configManager := &Manager{
+		configWatcher:        watcher,
+		multusConfig:         &config,
+		multusConfigDir:      multusConfigDir,
+		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
+		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+	}
+
+	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
+		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
+	}
+
+	return configManager, nil
+}
+
+func (m *Manager) loadPrimaryCNIConfigFromFile() error {
+	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
+	if err != nil {
+		return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
+	}
+	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
+}
+
+// OverrideNetworkName overrides the name of the multus configuration with the
+// name of the delegated primary CNI.
+func (m *Manager) OverrideNetworkName() error {
+	name, ok := m.cniConfigData["name"]
+	if !ok {
+		return fmt.Errorf("failed to access delegate CNI plugin name")
+	}
+	networkName := name.(string)
+
+	if networkName == "" {
+		return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
+	}
+	return m.multusConfig.Mutate(WithOverriddenName(networkName))
+}
+
+func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
+	cniConfigData := primaryCNIConfigData.(map[string]interface{})
+
+	m.cniConfigData = cniConfigData
+	return m.multusConfig.Mutate(
+		withDelegates(cniConfigData),
+		withCapabilities(cniConfigData))
+}
+
+// GenerateConfig generates a multus configuration from its current state
+func (m Manager) GenerateConfig() (string, error) {
+	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
+		return "", nil
+	}
+	return m.multusConfig.Generate()
+}
+
+// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
+// to by the primaryCNIPluginName attribute, and re-generates the multus
+// configuration whenever the primary CNI config is updated.
+func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
+	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
+
+	for {
+		select {
+		case event := <-m.configWatcher.Events:
+			// we're watching the DIR where the config sits, and the event
+			// does not concern the primary CNI config. Skip it.
+			if event.Name != m.primaryCNIConfigPath {
+				logging.Debugf("skipping un-related event %v", event)
+				continue
+			}
+
+			if !shouldRegenerateConfig(event) {
+				continue
+			}
+
+			updatedConfig, err := m.GenerateConfig()
+			if err != nil {
+				_ = logging.Errorf("failed to regenerate the multus configuration: %v", err)
+			}
+
+			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
+			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+			}
+			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+				_ = logging.Errorf("failed to reload the updated config: %v", err)
+			}
+
+		case err := <-m.configWatcher.Errors:
+			if err == nil {
+				continue
+			}
+			logging.Errorf("CNI monitoring error %v", err)
+
+		case <-shutDown:
+			logging.Verbosef("Stopped monitoring, closing channel ...")
+			_ = m.configWatcher.Close()
+			done <- struct{}{}
+			return nil
+		}
+	}
+}
+
+// PersistMultusConfig persists the provided configuration to the disc, with
+// Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
+func (m Manager) PersistMultusConfig(config string) error {
+	return ioutil.WriteFile(m.multusConfigFilePath, []byte(config), userRWPermission)
+}
+
+func primaryCNIPluginName(multusAutoconfigDir string) (string, error) {
+	masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
+	if err != nil {
+		return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
+	}
+	return masterCniConfigFileName, nil
+}
+
+func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
+	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
+}
+
+func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
+	}
+	defer func() {
+		// Close watcher on error
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+
+	if err = watcher.Add(cniConfigDir); err != nil {
+		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+	}
+
+	return watcher, nil
+}
+
+func shouldRegenerateConfig(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Write == fsnotify.Write ||
+		event.Op&fsnotify.Create == fsnotify.Create
+}
+
+func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
+	masterCNIConfigData, err := ioutil.ReadFile(masterCNIPluginPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
+	}
+
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall primary CNI config: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const suiteName = "Configuration Manager"
+
+func TestMultusConfigurationManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, suiteName)
+}
+
+var _ = Describe(suiteName, func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var configManager *Manager
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	BeforeEach(func() {
+		var err error
+		multusConfigDir, err = ioutil.TempDir("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+	})
+
+	BeforeEach(func() {
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), userRWPermission)).To(Succeed())
+
+		multusConf, _ := NewMultusConfig(
+			primaryCNIName,
+			cniVersion,
+			kubeconfig)
+		var err error
+		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
+		expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(config).To(Equal(expectedResult))
+	})
+
+	Context("Updates to the delegate CNI configuration", func() {
+		var (
+			doneChannel chan struct{}
+			stopChannel chan struct{}
+		)
+
+		BeforeEach(func() {
+			doneChannel = make(chan struct{})
+			stopChannel = make(chan struct{})
+			go func() {
+				Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
+			}()
+		})
+
+		AfterEach(func() {
+			go func() { stopChannel <- struct{}{} }()
+			Eventually(<-doneChannel).Should(Equal(struct{}{}))
+			close(doneChannel)
+			close(stopChannel)
+		})
+
+		It("Trigger the re-generation of the Multus CNI configuration", func() {
+			newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
+			Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), userRWPermission)).To(Succeed())
+
+			multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
+			Eventually(func() (string, error) {
+				multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
+				return string(multusCniData), err
+			}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
+		})
+	})
+
+	When("the user requests the name of the multus configuration to be overridden", func() {
+		BeforeEach(func() {
+			Expect(configManager.OverrideNetworkName()).To(Succeed())
+		})
+
+		It("Overrides the name of the multus configuration when requested", func() {
+			expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
+			config, err := configManager.GenerateConfig()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(expectedResult))
+		})
+	})
+})
+
+func multusConfigFromDelegate(delegateConfig string) string {
+	return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
+}

pkg/k8sclient/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2022 Multus Authors
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8sclient/k8sclient.go

@@ -1,5 +1,4 @@
-// Copyright (c) 2018 Intel Corporation
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2017 Intel Corporation
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,33 +17,33 @@ package k8sclient
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
-	"syscall"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
-	listers "k8s.io/client-go/listers/core/v1"
-	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/klog"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
-	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
-	netlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1"
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/kubeletclient"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/kubeletclient"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 )
 
 const (
@@ -61,13 +60,9 @@ type NoK8sNetworkError struct {
 // ClientInfo contains information given from k8s client
 type ClientInfo struct {
 	Client           kubernetes.Interface
-	NetClient        netclient.Interface
+	NetClient        netclient.K8sCniCncfIoV1Interface
 	EventBroadcaster record.EventBroadcaster
 	EventRecorder    record.EventRecorder
-
-	// multus-thick uses these informer
-	PodInformer    cache.SharedIndexInformer
-	NetDefInformer cache.SharedIndexInformer
 }
 
 // AddPod adds pod into kubernetes
@@ -77,27 +72,9 @@ func (c *ClientInfo) AddPod(pod *v1.Pod) (*v1.Pod, error) {
 
 // GetPod gets pod from kubernetes
 func (c *ClientInfo) GetPod(namespace, name string) (*v1.Pod, error) {
-	if c.PodInformer != nil {
-		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
-		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
-	}
 	return c.Client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
-// GetPodContext gets pod from kubernetes with context
-func (c *ClientInfo) GetPodContext(ctx context.Context, namespace, name string) (*v1.Pod, error) {
-	if c.PodInformer != nil {
-		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
-		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
-	}
-	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
-}
-
-// GetPodAPILiveQuery does a live API query for the pod, instead of using informers, for cases when a failure occurred, as to prevent a cache miss.
-func (c *ClientInfo) GetPodAPILiveQuery(ctx context.Context, namespace, name string) (*v1.Pod, error) {
-	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
-}
-
 // DeletePod deletes a pod from kubernetes
 func (c *ClientInfo) DeletePod(namespace, name string) error {
 	return c.Client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
@@ -105,16 +82,7 @@ func (c *ClientInfo) DeletePod(namespace, name string) error {
 
 // AddNetAttachDef adds net-attach-def into kubernetes
 func (c *ClientInfo) AddNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) (*nettypes.NetworkAttachmentDefinition, error) {
-	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
-}
-
-// GetNetAttachDef get net-attach-def from kubernetes
-func (c *ClientInfo) GetNetAttachDef(namespace, name string) (*nettypes.NetworkAttachmentDefinition, error) {
-	if c.NetDefInformer != nil {
-		logging.Debugf("GetNetAttachDef for [%s/%s] will use informer cache", namespace, name)
-		return netlister.NewNetworkAttachmentDefinitionLister(c.NetDefInformer.GetIndexer()).NetworkAttachmentDefinitions(namespace).Get(name)
-	}
-	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
 }
 
 // Eventf puts event into kubernetes events
@@ -128,34 +96,28 @@ func (e *NoK8sNetworkError) Error() string { return e.message }
 
 // SetNetworkStatus sets network status into Pod annotation
 func SetNetworkStatus(client *ClientInfo, k8sArgs *types.K8sArgs, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
-	podName := string(k8sArgs.K8S_POD_NAME)
-	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
-	podUID := string(k8sArgs.K8S_POD_UID)
-
-	return SetPodNetworkStatusAnnotation(client, podName, podNamespace, podUID, netStatus, conf)
-}
-
-// SetPodNetworkStatusAnnotation sets network status into Pod annotation
-func SetPodNetworkStatusAnnotation(client *ClientInfo, podName string, podNamespace string, podUID string, netStatus []nettypes.NetworkStatus, conf *types.NetConf) error {
 	var err error
-	logging.Debugf("SetPodNetworkStatusAnnotation: %v, %v, %v", client, netStatus, conf)
+	logging.Debugf("SetNetworkStatus: %v, %v, %v, %v", client, k8sArgs, netStatus, conf)
 
 	client, err = GetK8sClient(conf.Kubeconfig, client)
 	if err != nil {
 		return logging.Errorf("SetNetworkStatus: %v", err)
 	}
-	if client == nil {
+	if client == nil || client.Client == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
 			return logging.Errorf("SetNetworkStatus: must have either Kubernetes config or delegates")
 		}
-		logging.Debugf("SetPodNetworkStatusAnnotation: kube client info is not defined, skip network status setup")
+		logging.Debugf("SetNetworkStatus: kube client info is not defined, skip network status setup")
 		return nil
 	}
 
+	podName := string(k8sArgs.K8S_POD_NAME)
+	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podUID := string(k8sArgs.K8S_POD_UID)
 	pod, err := client.GetPod(podNamespace, podName)
 	if err != nil {
-		return logging.Errorf("SetPodNetworkStatusAnnotation: failed to query the pod %v in out of cluster comm: %v", podName, err)
+		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", podName, err)
 	}
 
 	if podUID != "" && string(pod.UID) != podUID && !IsStaticPod(pod) {
@@ -165,7 +127,7 @@ func SetPodNetworkStatusAnnotation(client *ClientInfo, podName string, podNamesp
 	if netStatus != nil {
 		err = netutils.SetNetworkStatus(client.Client, pod, netStatus)
 		if err != nil {
-			return logging.Errorf("SetPodNetworkStatusAnnotation: failed to update the pod %v in out of cluster comm: %v", podName, err)
+			return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", podName, err)
 		}
 	}
 
@@ -199,22 +161,16 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	// Check and see if each item matches the specification for valid attachment name.
 	// "Valid attachment names must be comprised of units of the DNS-1123 label format"
 	// [a-z0-9]([-a-z0-9]*[a-z0-9])?
+	// And we allow at (@), and forward slash (/) (units separated by commas)
 	// It must start and end alphanumerically.
-	allItems := []string{netNsName, networkName}
-	expr := regexp.MustCompile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
+	allItems := []string{netNsName, networkName, netIfName}
 	for i := range allItems {
-		matched := expr.MatchString(allItems[i])
+		matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i])
 		if !matched && len([]rune(allItems[i])) > 0 {
 			return "", "", "", logging.Errorf(fmt.Sprintf("parsePodNetworkObjectName: Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
 		}
 	}
 
-	if len(netIfName) > 0 {
-		if len(netIfName) > (syscall.IFNAMSIZ-1) || strings.ContainsAny(netIfName, " \t\n\v\f\r/") {
-			return "", "", "", logging.Errorf(fmt.Sprintf("parsePodNetworkObjectName: Failed to parse interface name: must be less than 15 chars and not contain '/' or spaces. interface name '%s'", netIfName))
-		}
-	}
-
 	logging.Debugf("parsePodNetworkObjectName: parsed: %s, %s, %s", netNsName, networkName, netIfName)
 	return netNsName, networkName, netIfName, nil
 }
@@ -227,7 +183,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 		return nil, logging.Errorf("parsePodNetworkAnnotation: pod annotation does not have \"network\" as key")
 	}
 
-	if strings.ContainsAny(podNetworks, "[{\"") {
+	if strings.IndexAny(podNetworks, "[{\"") >= 0 {
 		if err := json.Unmarshal([]byte(podNetworks), &networks); err != nil {
 			return nil, logging.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err)
 		}
@@ -291,8 +247,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 
 	logging.Debugf("getKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
-
-	customResource, err := client.GetNetAttachDef(net.Namespace, net.Name)
+	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(context.TODO(), net.Name, metav1.GetOptions{})
 	if err != nil {
 		errMsg := fmt.Sprintf("cannot find a network-attachment-definition (%s) in namespace (%s): %v", net.Name, net.Namespace, err)
 		if client != nil {
@@ -304,7 +259,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 	// Get resourceName annotation from NetworkAttachmentDefinition
 	deviceID := ""
 	resourceName, ok := customResource.GetAnnotations()[resourceNameAnnot]
-	if ok && pod != nil && pod.Name != "" && pod.Namespace != "" {
+	if ok && pod.Name != "" && pod.Namespace != "" {
 		// ResourceName annotation is found; try to get device info from resourceMap
 		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
 
@@ -410,7 +365,7 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 			}
 		}
 
-		if isGatewayConfigured {
+		if isGatewayConfigured == true {
 			err = types.CheckGatewayConfig(conf.Delegates)
 			if err != nil {
 				return 0, nil, err
@@ -426,6 +381,66 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 	return 0, clientInfo, err
 }
 
+// GetK8sClient gets client info from kubeconfig
+func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
+	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
+	// If we get a valid kubeClient (eg from testcases) just return that
+	// one.
+	if kubeClient != nil {
+		return kubeClient, nil
+	}
+
+	var err error
+	var config *rest.Config
+
+	// Otherwise try to create a kubeClient from a given kubeConfig
+	if kubeconfig != "" {
+		// uses the current context in kubeconfig
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
+		}
+	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
+		// Try in-cluster config where multus might be running in a kubernetes pod
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
+		}
+	} else {
+		// No kubernetes config; assume we shouldn't talk to Kube at all
+		return nil, nil
+	}
+
+	// Specify that we use gRPC
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+	// Set the config timeout to one minute.
+	config.Timeout = time.Minute
+
+	// creates the clientset
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	netclient, err := netclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
+
+	return &ClientInfo{
+		Client:           client,
+		NetClient:        netclient,
+		EventBroadcaster: broadcaster,
+		EventRecorder:    recorder,
+	}, nil
+}
+
 // GetPodNetwork gets net-attach-def annotation from pod
 func GetPodNetwork(pod *v1.Pod) ([]*types.NetworkSelectionElement, error) {
 	logging.Debugf("GetPodNetwork: %v", pod)
@@ -485,41 +500,32 @@ func isValidNamespaceReference(targetns string, allowednamespaces []string) bool
 	return false
 }
 
-// getNetDelegate loads delegate network for clusterNetwork/defaultNetworks
 func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 	logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace)
+	// option1) search CRD object for the network
+	net := &types.NetworkSelectionElement{
+		Name:      netname,
+		Namespace: namespace,
+	}
+	delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
+	if err == nil {
+		return delegate, resourceMap, nil
+	}
+
+	// option2) search CNI json config file
 	var configBytes []byte
-	isNetnamePath := strings.Contains(netname, "/")
-
-	// if netname is not directory or file, it must be net-attach-def name or CNI config name
-	if !isNetnamePath {
-		// option1) search CRD object for the network
-		net := &types.NetworkSelectionElement{
-			Name:      netname,
-			Namespace: namespace,
-		}
-		delegate, resourceMap, err := getKubernetesDelegate(client, net, confdir, pod, resourceMap)
-		if err == nil {
-			return delegate, resourceMap, nil
-		}
-
-		// option2) search CNI json config file, which has <netname> as CNI name, from confDir
-
-		configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
-		if err == nil {
-			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
-			if err != nil {
-				return nil, resourceMap, err
-			}
-			return delegate, resourceMap, nil
-		}
-	} else {
-		fInfo, err := os.Stat(netname)
+	configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
+	if err == nil {
+		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
 		if err != nil {
 			return nil, resourceMap, err
 		}
+		return delegate, resourceMap, nil
+	}
 
-		// option3) search directory
+	// option3) search directory
+	fInfo, err := os.Stat(netname)
+	if err == nil {
 		if fInfo.IsDir() {
 			files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
 			if err != nil {
@@ -537,122 +543,11 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 				}
 				return nil, resourceMap, err
 			}
-		} else {
-			// option4) if file path (absolute), then load it directly
-			if strings.HasSuffix(netname, ".conflist") {
-				confList, err := LoadChainedPluginsFromFile(netname)
-				if err != nil {
-					return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
-				}
-
-				delegate, err := types.LoadDelegateNetConfFromConfList(confList, nil, "", "")
-				if err != nil {
-					return nil, resourceMap, err
-				}
-				return delegate, resourceMap, nil
-
-			}
-
-			// Or it's not a conflist...
-			// after libcni v1.2.3 there's no support support this old-school method with non-conflists.
-			// this method doesn't check if there's a 0 length plugins field, that is.
-			conf, err := libcni.ConfFromFile(netname)
-			if err != nil {
-				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
-			}
-			if conf.Network.Type == "" {
-				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is supposed to be a .conflist?", netname)
-			}
-
-			delegate, err := types.LoadDelegateNetConf(conf.Bytes, nil, "", "")
-			if err != nil {
-				return nil, resourceMap, err
-			}
-			return delegate, resourceMap, nil
 		}
-
 	}
 	return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
 }
 
-func loadSubdirectoryChain(bytes []byte, cniconfdir string) (*libcni.NetworkConfigList, error) {
-	// Load the network configuration from the byte array
-	conf, err := libcni.NetworkConfFromBytes(bytes)
-	if err != nil {
-		return nil, fmt.Errorf("error loading network config from bytes: %v", err)
-	}
-
-	// Check if plugins need to be loaded from files
-	if !conf.LoadOnlyInlinedPlugins && cniconfdir != "" {
-		// Let's validate that conf.Name
-		// From the CNI spec:
-		// > Must start with an alphanumeric character, optionally followed by any combination of one or more alphanumeric characters,
-		// > underscore, dot (.) or hyphen (-). Must not contain characters disallowed in file paths.
-		if !regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`).MatchString(conf.Name) {
-			return nil, fmt.Errorf("invalid network config name: %s", conf.Name)
-		}
-
-		plugins, err := libcni.NetworkPluginConfsFromFiles(cniconfdir, conf.Name)
-		if err != nil {
-			return nil, fmt.Errorf("error loading plugin configs: %v", err)
-		}
-		conf.Plugins = append(conf.Plugins, plugins...)
-	}
-
-	if len(conf.Plugins) == 0 {
-		return nil, fmt.Errorf("no plugin configs found")
-	}
-
-	return conf, nil
-}
-
-// LoadChainedDelegatesFromBytes loads a CNI configuration byte array and returns a DelegateNetConf with the chain added.
-func LoadChainedDelegatesFromBytes(bytes []byte, cniconfdir string) *types.DelegateNetConf {
-	conf, err := loadSubdirectoryChain(bytes, cniconfdir)
-	if err != nil {
-		logging.Errorf("LoadChainedDelegatesFromBytes: %v", err)
-		return nil
-	}
-
-	// Create and return a DelegateNetConf from the configuration list
-	delegate, err := types.LoadDelegateNetConfFromConfList(conf, nil, "", "")
-	if err != nil {
-		logging.Errorf("LoadChainedDelegatesFromBytes: error loading delegate network config: %v", err)
-		return nil
-	}
-
-	return delegate
-}
-
-// LoadChainedPluginsFromFile loads a CNI configuration file and returns the NetworkConfigList
-func LoadChainedPluginsFromFile(filename string) (*libcni.NetworkConfigList, error) {
-	cleanPath := filepath.Clean(filename)
-
-	// stat the file to make sure it's a normal file.
-	info, err := os.Stat(cleanPath)
-	if err != nil {
-		return nil, err
-	}
-
-	if !info.Mode().IsRegular() {
-		return nil, errors.New("CNI configuration path is not a regular file")
-	}
-
-	bytes, err := os.ReadFile(cleanPath)
-	if err != nil {
-		return nil, fmt.Errorf("error reading %s: %w", filename, err)
-	}
-	logging.Debugf("LoadChainedPluginsFromFile: %s", filename)
-
-	conf, err := loadSubdirectoryChain(bytes, filepath.Dir(filename))
-	if err != nil {
-		return nil, err
-	}
-	logging.Debugf("Loaded SubdirectoryChain: %+v", conf)
-
-	return conf, nil
-}
-
 // GetDefaultNetworks parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
 func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo, resourceMap map[string]*types.ResourceInfo) (map[string]*types.ResourceInfo, error) {
 	logging.Debugf("GetDefaultNetworks: %v, %v, %v, %v", pod, conf, kubeClient, resourceMap)
@@ -679,7 +574,7 @@ func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo
 	delegates = append(delegates, delegate)
 
 	// Pod in kube-system namespace does not have default network for now.
-	if pod != nil && !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
+	if !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
 		for _, netname := range conf.DefaultNetworks {
 			delegate, resourceMap, err := getNetDelegate(kubeClient, pod, netname, conf.ConfDir, conf.MultusNamespace, resourceMap)
 			if err != nil {
@@ -731,7 +626,7 @@ const ConfigSourceAnnotationKey = "kubernetes.io/config.source"
 // IsStaticPod returns true if the pod is static pod.
 func IsStaticPod(pod *v1.Pod) bool {
 	if pod.Annotations != nil {
-		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok {
+		if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok == true {
 			return source != "api"
 		}
 	}

pkg/k8sclient/k8sclient_test.go

@@ -1,5 +1,4 @@
-// Copyright (c) 2018 Intel Corporation
-// Copyright (c) 2021 Multus Authors
+// Copyright (c) 2017 Intel Corporation
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,22 +11,22 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+//
 
 package k8sclient
 
-// disable dot-imports only for testing
-//revive:disable:dot-imports
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 
 	types020 "github.com/containernetworking/cni/pkg/types/020"
-	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
 
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
@@ -35,7 +34,7 @@ import (
 
 	"k8s.io/client-go/kubernetes/fake"
 
-	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
@@ -48,7 +47,7 @@ func TestK8sClient(t *testing.T) {
 func NewFakeClientInfo() *ClientInfo {
 	return &ClientInfo{
 		Client:    fake.NewSimpleClientset(),
-		NetClient: netfake.NewSimpleClientset(),
+		NetClient: netfake.NewSimpleClientset().K8sCniCncfIoV1(),
 	}
 }
 
@@ -61,7 +60,7 @@ var _ = Describe("k8sclient operations", func() {
 	const fakePodName string = "testPod"
 
 	BeforeEach(func() {
-		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		tmpDir, err = ioutil.TempDir("", "multus_tmp")
 		Expect(err).NotTo(HaveOccurred())
 		genericConf = `{
 			"name":"node-cni-network",
@@ -513,7 +512,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
 	})
 
-	It("retrieves cluster network from directory path", func() {
+	It("retrieves cluster network from path", func() {
 		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
@@ -545,37 +544,6 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
 	})
 
-	It("retrieves cluster network from cni config path", func() {
-		net1Name := filepath.Join(tmpDir, "10-net1.conf")
-		os.WriteFile(net1Name, []byte(`{
-				"name": "net1",
-				"type": "mynet",
-				"cniVersion": "0.3.1"
-			}`), 0600)
-
-		fakePod := testutils.NewFakePod(fakePodName, "", "")
-		conf := fmt.Sprintf(`{
-			"name":"node-cni-network",
-			"type":"multus",
-			"clusterNetwork": "%s",
-			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
-		}`, net1Name)
-		netConf, err := types.LoadNetConf([]byte(conf))
-		Expect(err).NotTo(HaveOccurred())
-
-		clientInfo := NewFakeClientInfo()
-		_, err = clientInfo.AddPod(fakePod)
-		Expect(err).NotTo(HaveOccurred())
-		_, err = GetK8sArgs(args)
-		Expect(err).NotTo(HaveOccurred())
-
-		_, err = GetDefaultNetworks(fakePod, netConf, clientInfo, nil)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(netConf.Delegates)).To(Equal(1))
-		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
-		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
-	})
-
 	It("Error in case of CRD not found", func() {
 		fakePod := testutils.NewFakePod(fakePodName, "", "")
 		conf := `{
@@ -766,7 +734,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("uses cached delegates when an error in loading from pod annotation occurs", func() {
-		dir, err := os.MkdirTemp("", "multus-test")
+		dir, err := ioutil.TempDir("", "multus-test")
 		Expect(err).NotTo(HaveOccurred())
 		defer os.RemoveAll(dir) // clean up
 
@@ -1002,30 +970,36 @@ users:
 	})
 
 	Context("parsePodNetworkObjectName", func() {
-		DescribeTable("fails to get podnetwork given bad annotation values", func(networkAnnot string) {
-			pod := testutils.NewFakePod(fakePodName, "net1", "")
-			pod.Annotations[networkAttachmentAnnot] = networkAnnot
+		It("fails to get podnetwork given bad annotation values", func() {
+			fakePod := testutils.NewFakePod(fakePodName, "net1", "")
+
+			clientInfo := NewFakeClientInfo()
+			_, err := clientInfo.AddPod(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = clientInfo.AddNetAttachDef(
+				testutils.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", "{\"type\": \"mynet\"}"))
+			Expect(err).NotTo(HaveOccurred())
+
+			k8sArgs, err := GetK8sArgs(args)
+			Expect(err).NotTo(HaveOccurred())
+			pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+
+			// invalid case 1 - can't have more than 2 items separated by "/"
+			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP/root@thirdIP"
 			_, err = GetPodNetwork(pod)
 			Expect(err).To(HaveOccurred())
-		},
-			Entry("can't have more than 2 items separated by \"/\"", "root@someIP/root@someOtherIP/root@thirdIP"),
-			Entry("can't have more than 2 items separated by \"@\"", "root@someIP/root@someOtherIP@garbagevalue"),
-			Entry("not matching comma-delimited format", "root@someIP/root@someOtherIP"),
-			Entry("invalid network interface name space in netdev name", "default/net1@myIfc Name"),
-			Entry("invalid network interface name too long", "default/net1@very_long_interface_name"),
-		)
 
-		DescribeTable("gets pod network successfully from annotation values", func(networkAnnot string) {
-			pod := testutils.NewFakePod(fakePodName, "net1", "")
-			pod.Annotations[networkAttachmentAnnot] = networkAnnot
+			// invalid case 2 - can't have more than 2 items separated by "@"
+			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP@garbagevalue"
 			_, err = GetPodNetwork(pod)
-			Expect(err).ToNot(HaveOccurred())
-		},
-			Entry("network without namespace", "net1"),
-			Entry("network with namespace", "default/net1"),
-			Entry("network with interface name", "net1@my_interface"),
-			Entry("network with interface name and namespace", "default/net1@my_interface"),
-		)
+			Expect(err).To(HaveOccurred())
+
+			// invalid case 3 - not matching comma-delimited format
+			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP"
+			_, err = GetPodNetwork(pod)
+			Expect(err).To(HaveOccurred())
+		})
 	})
 
 	Context("setPodNetworkAnnotation", func() {
@@ -1199,14 +1173,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1257,14 +1228,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1318,14 +1286,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1403,14 +1368,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1462,14 +1424,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1520,14 +1479,11 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
+			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := make([]nettypes.NetworkStatus, 0)
-			for _, status := range delegateNetStatuses {
-				netstatus = append(netstatus, *status)
-			}
+			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 

pkg/k8sclient/kubeconfig.go

@@ -1,247 +0,0 @@
-// Copyright (c) 2023 Multus Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package k8sclient
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"fmt"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	certificatesv1 "k8s.io/api/certificates/v1"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/client-go/transport"
-	"k8s.io/client-go/util/certificate"
-	"k8s.io/klog"
-
-	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-)
-
-const (
-	certNamePrefix       = "multus-client"
-	certCommonNamePrefix = "system:multus"
-	certOrganization     = "system:multus"
-)
-
-var (
-	certUsages = []certificatesv1.KeyUsage{certificatesv1.UsageDigitalSignature, certificatesv1.UsageClientAuth}
-)
-
-// getPerNodeKubeconfig creates new kubeConfig, based on bootstrap, with new certDir
-func getPerNodeKubeconfig(bootstrap *rest.Config, certDir string) *rest.Config {
-	return &rest.Config{
-		Host:    bootstrap.Host,
-		APIPath: bootstrap.APIPath,
-		ContentConfig: rest.ContentConfig{
-			AcceptContentTypes: "application/vnd.kubernetes.protobuf,application/json",
-			ContentType:        "application/vnd.kubernetes.protobuf",
-		},
-		TLSClientConfig: rest.TLSClientConfig{
-			KeyFile:  path.Join(certDir, certNamePrefix+"-current.pem"),
-			CertFile: path.Join(certDir, certNamePrefix+"-current.pem"),
-			CAData:   bootstrap.TLSClientConfig.CAData,
-		},
-		// Allow multus (especially in server mode) to make more concurrent requests
-		// to reduce client-side throttling
-		QPS:   50,
-		Burst: 50,
-		// Set the config timeout to one minute.
-		Timeout: time.Minute,
-	}
-}
-
-// PerNodeK8sClient creates/reload new multus kubeconfig per-node.
-func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration time.Duration, certDir string) (*ClientInfo, error) {
-	bootstrapKubeconfig, err := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
-	if err != nil {
-		return nil, logging.Errorf("failed to load bootstrap kubeconfig %s: %v", bootstrapKubeconfigFile, err)
-	}
-	config := getPerNodeKubeconfig(bootstrapKubeconfig, certDir)
-
-	// If we have a valid certificate, user that to fetch CSRs.
-	// Otherwise, use the bootstrap credentials from bootstrapKubeconfig
-	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
-	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
-		cfg := bootstrapKubeconfig
-
-		// validate the kubeconfig
-		tempClient, err := kubernetes.NewForConfig(cfg)
-		if err != nil {
-			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
-		} else {
-			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
-			// tls unknown authority error is unrecoverable error with retry
-			if err != nil {
-				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
-					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
-					// reload and use bootstrapKubeconfig again
-					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
-					cfg = newBootstrapKubeconfig
-				} else {
-					logging.Errorf("failed to list pods with new certs: %v", err)
-				}
-			}
-
-			if current != nil {
-				cfg = config
-			}
-		}
-		return kubernetes.NewForConfig(cfg)
-	}
-
-	certificateStore, err := certificate.NewFileStore(certNamePrefix, certDir, certDir, "", "")
-	if err != nil {
-		return nil, logging.Errorf("failed to initialize the certificate store: %v", err)
-	}
-
-	certManager, err := certificate.NewManager(&certificate.Config{
-		ClientsetFn: newClientsetFn,
-		Template: &x509.CertificateRequest{
-			Subject: pkix.Name{
-				CommonName:   fmt.Sprintf("%s:%s", certCommonNamePrefix, nodeName),
-				Organization: []string{certOrganization},
-			},
-		},
-		RequestedCertificateLifetime: &certDuration,
-		SignerName:                   certificatesv1.KubeAPIServerClientSignerName,
-		Usages:                       certUsages,
-		CertificateStore:             certificateStore,
-	})
-	if err != nil {
-		return nil, logging.Errorf("failed to initialize the certificate manager: %v", err)
-	}
-	if certDuration < time.Hour {
-		// the default value for CertCallbackRefreshDuration (5min) is too long for short-lived certs,
-		// set it to a more sensible value
-		transport.CertCallbackRefreshDuration = time.Second * 10
-	}
-	certManager.Start()
-
-	logging.Verbosef("Waiting for certificate")
-	var storeErr error
-	err = wait.PollWithContext(context.TODO(), time.Second, 2*time.Minute, func(_ context.Context) (bool, error) {
-		var currentCert *tls.Certificate
-		currentCert, storeErr = certificateStore.Current()
-		return currentCert != nil && storeErr == nil, nil
-	})
-	if err != nil {
-		return nil, logging.Errorf("certificate was not signed, last cert store err: %v err: %v", storeErr, err)
-	}
-	logging.Verbosef("Certificate found!")
-
-	return newClientInfo(config)
-}
-
-// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
-// the k8s API.
-func InClusterK8sClient() (*ClientInfo, error) {
-	clientInfo, err := GetK8sClient("", nil)
-	if err != nil {
-		return nil, err
-	}
-	if clientInfo == nil {
-		return nil, fmt.Errorf("failed to create in-cluster kube client")
-	}
-	return clientInfo, err
-}
-
-// SetK8sClientInformers adds informer structure to ClientInfo to utilize in thick daemon
-func (c *ClientInfo) SetK8sClientInformers(podInformer, netDefInformer cache.SharedIndexInformer) {
-	c.PodInformer = podInformer
-	c.NetDefInformer = netDefInformer
-}
-
-// GetK8sClient gets client info from kubeconfig
-func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
-	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
-	// If we get a valid kubeClient (eg from testcases) just return that
-	// one.
-	if kubeClient != nil {
-		return kubeClient, nil
-	}
-
-	var err error
-	var config *rest.Config
-
-	// Otherwise try to create a kubeClient from a given kubeConfig
-	if kubeconfig != "" {
-		// uses the current context in kubeconfig
-		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
-		}
-	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
-		// Try in-cluster config where multus might be running in a kubernetes pod
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
-		}
-	} else {
-		// No kubernetes config; assume we shouldn't talk to Kube at all
-		return nil, nil
-	}
-
-	// Specify that we use gRPC
-	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
-	config.ContentType = "application/vnd.kubernetes.protobuf"
-	// Set the config timeout to one minute.
-	config.Timeout = time.Minute
-	// Allow multus (especially in server mode) to make more concurrent requests
-	// to reduce client-side throttling
-	config.QPS = 50
-	config.Burst = 50
-
-	return newClientInfo(config)
-}
-
-// newClientInfo returns a `ClientInfo` from a configuration created from an
-// existing kubeconfig file.
-func newClientInfo(config *rest.Config) (*ClientInfo, error) {
-	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	netclient, err := netclient.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(klog.Infof)
-	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
-	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
-	return &ClientInfo{
-		Client:           client,
-		NetClient:        netclient,
-		EventBroadcaster: broadcaster,
-		EventRecorder:    recorder,
-	}, nil
-}

pkg/kubeletclient/doc.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2022 Multus Authors
+// Copyright (c) 2021 Multus Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.