Merge pull request #1424 from nirdothan/mount-multus-conf-dir

Thick client: mount multus-conf-dir

.github/workflows/build.yml

@@ -4,18 +4,18 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go-version: [1.22.x, 1.23.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Build
       env:

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/image-build.yml

@@ -6,14 +6,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
@@ -25,50 +25,72 @@ jobs:
 
       # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container debug image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
-  build-amd64-thick:
-    name: Image build/amd64 thick plugin
+  build-thick:
+    name: Image thick plugin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build container image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:latest-amd64-thick
+          tags: ghcr.io/${{ github.repository }}:latest-thick
           file: images/Dockerfile.thick
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@0.29.0
+        with:
+          image-ref: ghcr.io/${{ github.repository }}:latest-thick
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
 
   build-origin:
     name: Image build/origin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Download OKD Builder Dockerfile
-        run: curl https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile -o images/okd-builder.Dockerfile
+        run: curl https://raw.githubusercontent.com/okd-project/images/main/builder/Dockerfile -o images/okd-builder.Dockerfile
 
       - name: Patch OKD Builder Dockerfile to workaround error
         run: sed -i -e "s/yum install -y yum-utils/rpm --import \/etc\/pki\/rpm-gpg\/*;yum install -y yum-utils/" images/okd-builder.Dockerfile
 
+      - name: Create root for builder
+        run: mkdir root
+
       - name: Organically build golang builder image
         run: docker build -t local/okdbuilder:latest -f images/okd-builder.Dockerfile .
 

.github/workflows/image-push-master.yml

@@ -1,24 +1,24 @@
 name: Image push for master
-on: 
+on:
   push:
     branches:
       - master
 env:
   image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick-amd64:
-    name: Image push thick image/amd64
+  push-thick:
+    name: Image push thick image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -26,7 +26,7 @@ jobs:
 
       - name: Push container image for thick plugin
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -34,21 +34,23 @@ jobs:
             ghcr.io/${{ github.repository }}:latest-thick
             ghcr.io/${{ github.repository }}:snapshot-thick
           file: images/Dockerfile.thick
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
   push-thin:
     name: Image push thin image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -56,7 +58,7 @@ jobs:
 
       - name: Push thin container image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -64,13 +66,13 @@ jobs:
             ghcr.io/${{ github.repository }}:latest
             ghcr.io/${{ github.repository }}:snapshot
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
       - name: Push thin container debug image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -78,7 +80,7 @@ jobs:
             ghcr.io/${{ github.repository }}:latest-debug
             ghcr.io/${{ github.repository }}:snapshot-debug
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
@@ -88,14 +90,14 @@ jobs:
 #    runs-on: ubuntu-latest
 #    steps:
 #      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v3
+#        uses: actions/checkout@v4
 #
 #      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v2
+#        uses: docker/setup-buildx-action@v3
 #
 #      - name: Login to GitHub Container Registry
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v2
+#        uses: docker/login-action@v3
 #        with:
 #          registry: ghcr.io
 #          username: ${{ github.repository_owner }}
@@ -103,7 +105,7 @@ jobs:
 #
 #      - name: Push container image
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v3
+#        uses: docker/build-push-action@v5
 #        with:
 #          context: .
 #          push: true

.github/workflows/image-push-release.yml

@@ -1,24 +1,24 @@
 name: Image push release
-on: 
+on:
   push:
     tags:
       - v*
 env:
   image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick-amd64:
-    name: Image push thick image/amd64
+  push-thick:
+    name: Image push thick image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -26,7 +26,7 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/${{ github.repository }}
           flavor: |
@@ -34,7 +34,7 @@ jobs:
 
       - name: Push container image for thick plugin
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -42,21 +42,23 @@ jobs:
             ghcr.io/${{ github.repository }}:stable-thick
             ${{ steps.docker_meta.outputs.tags }}-thick
           file: images/Dockerfile.thick
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
   push-thin:
     name: Image push thin image/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -64,7 +66,7 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/${{ github.repository }}
           flavor: |
@@ -72,7 +74,7 @@ jobs:
 
       - name: Push thin container image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -80,13 +82,13 @@ jobs:
             ghcr.io/${{ github.repository }}:stable
             ${{ steps.docker_meta.outputs.tags }}
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
       - name: Push thin container debug image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -94,7 +96,7 @@ jobs:
             ghcr.io/${{ github.repository }}:stable-debug
             ${{ steps.docker_meta.outputs.tags }}-debug
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
@@ -104,14 +106,14 @@ jobs:
 #    runs-on: ubuntu-latest
 #    steps:
 #      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v3
+#        uses: actions/checkout@v4
 #
 #      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v1
+#        uses: docker/setup-buildx-action@v3
 #
 #      - name: Login to GitHub Container Registry
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v1
+#        uses: docker/login-action@v3
 #        with:
 #          registry: ghcr.io
 #          username: ${{ github.repository_owner }}
@@ -126,7 +128,7 @@ jobs:
 #
 #      - name: Push container image
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v2
+#        uses: docker/build-push-action@v5
 #        with:
 #          context: .
 #          push: true

.github/workflows/kind-e2e.yml

@@ -25,32 +25,31 @@ jobs:
 #          - docker-file: images/Dockerfile
 #            cni-version: "1.0.0"
 #            multus-manifest: multus-daemonset.yml
+    env:
+      JOB_NAME: "${{ matrix.cni-version }}-${{ matrix.multus-manifest }}"
 
     if: >
       (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
         github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
 
       - name: Setup j2cli
         run: |
-          pip3 install --user --upgrade j2cli
-          j2 --version
-
-      - name: Setup registry
-        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+          sudo apt-get install -y j2cli
+          echo $(j2 --version)
 
       - name: Build latest-amd64
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           load: true
@@ -58,10 +57,6 @@ jobs:
           file: ${{ matrix.docker-file }}
           platforms: linux/amd64
 
-      # docker buildx push is failed due to https://github.com/docker/buildx/issues/94
-      - name: Push to local registry
-        run: docker push localhost:5000/multus:e2e
-
       - name: Get kind/kubectl/koko
         working-directory: ./e2e
         run: ./get_tools.sh
@@ -72,7 +67,7 @@ jobs:
 
       - name: Setup cluster
         working-directory: ./e2e
-        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} ./setup_cluster.sh
+        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} MULTUS_DOCKERFILE=none ./setup_cluster.sh
 
       - name: Test simple pod
         working-directory: ./e2e
@@ -90,8 +85,33 @@ jobs:
         working-directory: ./e2e
         run: ./test-default-route1.sh
 
+#      - name: Test DRA integration
+#        working-directory: ./e2e
+#        run: ./test-dra-integration.sh
+
+      - name: Test subdirectory CNI chaining
+        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        working-directory: ./e2e
+        run: ./test-subdirectory-chaining.sh
+
+      - name: Test subdirectory CNI chaining with passthru CNI / auxiliaryCNIChainName
+        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        working-directory: ./e2e
+        run: ./test-subdirectory-chaining-passthru.sh
+
+      - name: Export kind logs
+        if: always()
+        run: |
+          mkdir -p /tmp/kind/logs
+          kind export logs /tmp/kind/logs -v 2147483647
+
+      - name: Upload kind logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
+          path: /tmp/kind/logs
+
       - name: cleanup cluster and registry
         run: |
           kind delete cluster
-          docker kill kind-registry
-          docker rm kind-registry

.github/workflows/release.yml

@@ -8,17 +8,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: 1.22.x
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v4
+        uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist

.github/workflows/stale-issues-prs.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'

.github/workflows/test.yml

@@ -4,17 +4,17 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go-version: [1.22.x, 1.23.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run Revive Action by pulling pre-built image
       uses: docker://morphy/revive-action:v2

.gitignore

@@ -2,6 +2,7 @@
 bin/
 e2e/bin/
 e2e/yamls/
+e2e/repos/
 
 # GOPATH created by the build script
 gopath/

.travis.yml

@@ -1,116 +0,0 @@
-os: linux
-language: go
-# see https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments
-# for the detail
-# sudo: requried
-dist: bionic
-
-services:
-  - docker
-
-go:
-  - 1.13.x
-
-env:
-  global:
-    - GO111MODULE=on
-    - REGISTRY_USER=${REGISTRY_USER:-nfvpe}
-    - REGISTRY_PASS=${REGISTRY_PASS}
-    - REPOSITORY_NAME=${REPOSITORY_NAME}
-    - REPOSITORY_USER=${REPOSITORY_USER}
-    - DOCKER_CLI_EXPERIMENTAL="enabled"
-    - secure: "${REGISTRY_SECURE}"
-  jobs:
-    - TARGET=amd64
-    - TARGET=ppc64le
-
-before_install:
-  - if [ "${REPOSITORY_NAME}" = "" ]; then export REPOSITORY_NAME=multus; fi
-  - sudo apt-get update -qq
-  - go get github.com/mattn/goveralls
-
-install:
-  - go get -u golang.org/x/lint/golint
-
-before_script:
-  # Make gopath... to run golint/go fmt/go vet
-  # Suppress golint for fixing lint later.
-  - golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
-  - go fmt ./...
-  - go vet ./...
-#  - gocyclo -over 15 ./multus
-
-script:
-  - GOARCH="${TARGET}" ./hack/build-go.sh
-  - |
-    if [ "${TARGET}" == "amd64" ]; then
-      sudo env PATH=${PATH} ./scripts/test.sh
-      goveralls -coverprofile=coverage.out -service=travis-ci
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 .
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le -f Dockerfile.ppc64le .
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}-origin:latest -f Dockerfile.openshift .
-    fi
-
-deploy:
-  # Release on versioned tag (e.g. v1.0)
-  - provider: script
-    #cleanup: false
-    script: curl -sL https://git.io/goreleaser
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TARGET = amd64 && $TRAVIS_TAG =~ ^v[0-9].*$ && ! -z $GITHUB_TOKEN && $TRAVIS_OS_NAME = linux"
-  # Push images to Dockerhub on tag
-  - provider: script
-    cleanup: false
-    script: >
-      bash -c '
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
-      export DOCKER_CLI_EXPERIMENTAL="enabled";
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
-      echo done'
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TRAVIS_TAG =~ ^v[0-9].*$ && -n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
-  # Push images to Dockerhub on merge to master
-  - provider: script
-    on:
-      branch: master
-      condition: "-n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
-    script: >
-      bash -c '
-      docker tag ${REPOSITORY_USER}/:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      echo done'

README.md

@@ -24,10 +24,10 @@ Here's an illustration of the network interfaces attached to a pod, as provision
 
 The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugin, refer to our [quick-start guide](docs/quickstart.md).
 
-Clone this GitHub repository, and apply a daemonset which installs Multus using `kubectl`. From the root directory of the clone, apply the daemonset YAML file:
+To use latest features try command below which applies a daemonset and installs thick Multus using `kubectl`:
 
 ```
-cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
@@ -39,7 +39,7 @@ With the multus 4.0 release, we introduce a new client/server-style plugin deplo
 We recommend using the thick plugin in most environments, but if you wish to run the thin plugin, or are in a resource-constrained environment, you may do so with:
 
 ```
-cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 ```
 
 ## Additional Installation Options
@@ -62,3 +62,5 @@ In addition to the [quick-start guide](docs/quickstart.md), you may:
 ## Contact Us
 
 For any questions about Multus CNI, open up a GitHub issue or feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/).
+
+To be invited, use [this slack invite link](https://join.slack.com/t/npwg-team/shared_invite/zt-1u2vmsn2b-tKdOokdPY73zn9B32JoAOg).

cmd/cert-approver/main.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2023 Network Plumbing Working Group
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is Kubernetes controller which approves CSR submitted by multus.
+// This command is required only if multus runs with per-node certificate.
+package main
+
+// Note: cert-approver should be simple, just approve multus' CSR, hence
+// this go code should not have any dependencies from pkg/, if possible,
+// to keep its code simplicity.
+import (
+	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"os/signal"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/certificate/csr"
+	"k8s.io/client-go/util/workqueue"
+)
+
+// CertController object
+type CertController struct {
+	clientset          kubernetes.Interface
+	queue              workqueue.RateLimitingInterface
+	informer           cache.SharedIndexInformer
+	broadcaster        record.EventBroadcaster
+	recorder           record.EventRecorder
+	commonNamePrefixes string
+}
+
+const (
+	maxDuration                = time.Hour * 24 * 365
+	resyncPeriod time.Duration = time.Second * 3600 // resync every one hour, default is 10 hour
+	maxRetries                 = 5
+)
+
+var (
+	// ControllerName provides controller name
+	ControllerName = "csr-approver"
+	// NamePrefix specifies which name in certification request should be target to approve
+	NamePrefix = "system:multus"
+	// Organization specifies which org in certification request should be target to approve
+	Organization = []string{"system:multus"}
+	// Groups specifies which group in certification request should be target to approve
+	Groups = sets.New[string]("system:nodes", "system:multus", "system:authenticated")
+	// UserPrefixes specifies which name prefix in certification request should be target to approve
+	UserPrefixes = sets.New[string]("system:node", NamePrefix)
+	// Usages specifies which usage in certification request should be target to approve
+	Usages = sets.New[certificatesv1.KeyUsage](
+		certificatesv1.UsageDigitalSignature,
+		certificatesv1.UsageClientAuth)
+)
+
+// NewCertController creates certcontroller
+func NewCertController() (*CertController, error) {
+	var clientset kubernetes.Interface
+	// setup Kubernetes API client
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientset, err = kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	informer := cache.NewSharedIndexInformer(
+		cache.NewListWatchFromClient(
+			clientset.CertificatesV1().RESTClient(),
+			"certificatesigningrequests", corev1.NamespaceAll, fields.Everything()),
+		&certificatesv1.CertificateSigningRequest{},
+		resyncPeriod,
+		nil)
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientset.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "cert-approver"})
+	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+	c := &CertController{
+		clientset:          clientset,
+		informer:           informer,
+		queue:              queue,
+		commonNamePrefixes: NamePrefix,
+		broadcaster:        broadcaster,
+		recorder:           recorder,
+	}
+
+	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			if csr, ok := obj.(*certificatesv1.CertificateSigningRequest); ok {
+				if c.filterCSR(csr) {
+					key, err := cache.MetaNamespaceKeyFunc(obj)
+					if err == nil {
+						queue.Add(key)
+					}
+				}
+			}
+		},
+	})
+
+	return c, nil
+}
+
+// Run starts controller
+func (c *CertController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting cert approver")
+
+	go c.informer.Run(stopCh)
+	if !cache.WaitForCacheSync(stopCh, c.HasSynced) {
+		utilruntime.HandleError(fmt.Errorf("Timed out waiting for caches to sync"))
+		return
+	}
+
+	klog.Info("cert approver synced and ready")
+	wait.Until(c.runWorker, time.Second, stopCh)
+}
+
+// HasSynced is required for the cache.Controller interface.
+func (c *CertController) HasSynced() bool {
+	return c.informer.HasSynced()
+}
+
+// LastSyncResourceVersion is required for the cache.Controller interface.
+func (c *CertController) LastSyncResourceVersion() string {
+	return c.informer.LastSyncResourceVersion()
+}
+
+func (c *CertController) runWorker() {
+	for c.processNextItem() {
+		// continue looping
+	}
+}
+
+func (c *CertController) processNextItem() bool {
+	// Wait until there is a new item in the working queue
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	// Tell the queue that we are done with processing this key. This unblocks the key for other workers
+	// This allows safe parallel processing because two pods with the same key are never processed in
+	// parallel.
+	defer c.queue.Done(key)
+
+	// Invoke the method containing the business logic
+	err := c.processItem(key.(string))
+	// Handle the error if something went wrong during the execution of the business logic
+	c.handleErr(err, key)
+	return true
+
+}
+
+// handleErr checks if an error happened and makes sure we will retry later.
+func (c *CertController) handleErr(err error, key interface{}) {
+	if err == nil {
+		// Forget about the #AddRateLimited history of the key on every successful synchronization.
+		// This ensures that future processing of updates for this key is not delayed because of
+		// an outdated error history.
+		c.queue.Forget(key)
+		return
+	}
+
+	// This controller retries 5 times if something goes wrong. After that, it stops trying.
+	if c.queue.NumRequeues(key) < maxRetries {
+		klog.Infof("Error syncing csr %s: %v", key, err)
+		// Re-enqueue the key rate limited. Based on the rate limiter on the
+		// queue and the re-enqueue history, the key will be processed later again.
+		c.queue.AddRateLimited(key)
+		return
+	}
+
+	c.queue.Forget(key)
+	// Report to an external entity that, even after several retries, we could not successfully process this key
+	utilruntime.HandleError(err)
+	klog.Infof("Dropping csr %q out of the queue: %v", key, err)
+}
+
+func (c *CertController) processItem(key string) error {
+	startTime := time.Now()
+
+	obj, _, err := c.informer.GetIndexer().GetByKey(key)
+	if err != nil {
+		return fmt.Errorf("Error fetching object with key %s from store: %v", key, err)
+	}
+
+	req, _ := obj.(*certificatesv1.CertificateSigningRequest)
+
+	nodeName := "unknown"
+	defer func() {
+		klog.Infof("Finished syncing CSR %s for %s node in %v", req.Name, nodeName, time.Since(startTime))
+	}()
+
+	if len(req.Status.Certificate) > 0 {
+		klog.V(5).Infof("CSR %s is already signed", req.Name)
+		return nil
+	}
+
+	if isApprovedOrDenied(&req.Status) {
+		klog.V(5).Infof("CSR %s is already approved/denied", req.Name)
+		return nil
+	}
+
+	csrPEM, _ := pem.Decode(req.Spec.Request)
+	if csrPEM == nil {
+		return fmt.Errorf("failed to PEM-parse the CSR block in .spec.request: no CSRs were found")
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse the CSR bytes: %v", err)
+	}
+
+	i := strings.LastIndex(req.Spec.Username, ":")
+	if i == -1 || i == len(req.Spec.Username)-1 {
+		return fmt.Errorf("failed to parse the username: %s", req.Spec.Username)
+	}
+
+	ctx := context.Background()
+	prefix := req.Spec.Username[:i]
+	nodeName = req.Spec.Username[i+1:]
+	if !UserPrefixes.Has(prefix) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by an unexpected user: %q", req.Name, req.Spec.Username))
+	}
+
+	if errs := validation.IsDNS1123Subdomain(nodeName); len(errs) != 0 {
+		return c.denyCSR(ctx, req, fmt.Sprintf("extracted node name %q is not a valid DNS subdomain %v", nodeName, errs))
+	}
+
+	if usages := sets.New[certificatesv1.KeyUsage](req.Spec.Usages...); !usages.Equal(Usages) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with unexpected usages: %v", req.Name, usages.UnsortedList()))
+	}
+
+	if !Groups.HasAll(req.Spec.Groups...) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by a user with unexpected groups: %v", req.Name, req.Spec.Groups))
+	}
+
+	expectedSubject := fmt.Sprintf("%s:%s", c.commonNamePrefixes, nodeName)
+	if x509CSR.Subject.CommonName != expectedSubject {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's commonName to be %q, but it is %q", expectedSubject, x509CSR.Subject.CommonName))
+	}
+
+	if !reflect.DeepEqual(x509CSR.Subject.Organization, Organization) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's organization to be %v, but it is %v", Organization, x509CSR.Subject.Organization))
+	}
+
+	if req.Spec.ExpirationSeconds == nil {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created without specyfying the expirationSeconds", req.Name))
+	}
+
+	if csr.ExpirationSecondsToDuration(*req.Spec.ExpirationSeconds) > maxDuration {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with invalid expirationSeconds value: %d", req.Name, *req.Spec.ExpirationSeconds))
+	}
+
+	return c.approveCSR(ctx, req)
+}
+
+// CSR specific functions
+
+func (c *CertController) filterCSR(csr *certificatesv1.CertificateSigningRequest) bool {
+	nsName := types.NamespacedName{Namespace: csr.Namespace, Name: csr.Name}
+	csrPEM, _ := pem.Decode(csr.Spec.Request)
+	if csrPEM == nil {
+		klog.Errorf("Failed to PEM-parse the CSR block in .spec.request: no CSRs were found in %s", nsName)
+		return false
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		klog.Errorf("Failed to parse the CSR .spec.request of %q: %v", nsName, err)
+		return false
+	}
+
+	return strings.HasPrefix(x509CSR.Subject.CommonName, c.commonNamePrefixes) &&
+		csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName
+}
+
+func (c *CertController) approveCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateApproved,
+			Status:  corev1.ConditionTrue,
+			Reason:  "AutoApproved",
+			Message: fmt.Sprintf("Auto-approved CSR %q", csr.Name),
+		})
+
+	c.recorder.Eventf(csr, corev1.EventTypeNormal, "CSRApproved", "CSR %q has been approved by %s", csr.Name, ControllerName)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func (c *CertController) denyCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, message string) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateDenied,
+			Status:  corev1.ConditionTrue,
+			Reason:  "CSRDenied",
+			Message: message,
+		},
+	)
+
+	c.recorder.Eventf(csr, corev1.EventTypeWarning, "CSRDenied", "The CSR %q has been denied by: %s", csr.Name, ControllerName, message)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().Update(ctx, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func isApprovedOrDenied(status *certificatesv1.CertificateSigningRequestStatus) bool {
+	for _, c := range status.Conditions {
+		if c.Type == certificatesv1.CertificateApproved || c.Type == certificatesv1.CertificateDenied {
+			return true
+		}
+	}
+	return false
+}
+
+func main() {
+	klog.Infof("starting cert-approver")
+
+	// Start watching for pod creations
+	certController, err := NewCertController()
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+	go certController.Run(stopCh)
+
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+}

cmd/install_multus/main.go

@@ -53,4 +53,15 @@ func main() {
 	}
 
 	fmt.Printf("multus %s copy succeeded!\n", multusFileName)
+
+	// Copy the passthru CNI
+	passthruPath := "/usr/src/multus-cni/bin/passthru"
+	err = cmdutils.CopyFileAtomic(passthruPath, *destDir, fmt.Sprintf("%s.temp", "passthru"), "passthru")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("passthru cni %s copy succeeded!\n", passthruPath)
+
 }

cmd/kubeconfig_generator/main.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary submit CSR for kube controll access for multus thin plugin
+// and generate Kubeconfig
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"text/template"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+var kubeConfigTemplate = `apiVersion: v1
+clusters:
+  - cluster:
+      certificate-authority-data: {{.CADATA}}
+      server: {{.K8S_APISERVER}}
+    name: default-cluster
+contexts:
+  - context:
+      cluster: default-cluster
+      namespace: default
+      user: default-auth
+    name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+  - name: default-auth
+    user:
+      client-certificate: {{.CERTDIR}}/multus-client-current.pem
+      client-key: {{.CERTDIR}}/multus-client-current.pem
+`
+
+func main() {
+	certDir := pflag.StringP("certdir", "", "/tmp", "specify cert directory")
+	bootstrapConfig := pflag.StringP("bootstrap-config", "", "/tmp/kubeconfig", "specify bootstrap kubernetes config")
+	kubeconfigPathRaw := pflag.StringP("kubeconfig", "", "/run/multus/kubeconfig", "specify output kubeconfig path")
+	certDurationString := pflag.StringP("cert-duration", "", "10m", "specify certificate duration")
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	kubeconfigPath, err := filepath.Abs(*kubeconfigPathRaw)
+	if err != nil {
+		klog.Fatalf("illegal path %s in kubeconfigPath %s: %v", kubeconfigPath, *kubeconfigPathRaw, err)
+	}
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	// check variables
+	if _, err := os.Stat(*bootstrapConfig); err != nil {
+		klog.Fatalf("failed to read bootstrap config %q", *bootstrapConfig)
+	}
+	st, err := os.Stat(*certDir)
+	if err != nil {
+		klog.Fatalf("failed to find cert directory %q", *certDir)
+	}
+	if !st.IsDir() {
+		klog.Fatalf("cert directory %q is not directory", *certDir)
+	}
+	certDuration, err := time.ParseDuration(*certDurationString)
+	if err != nil {
+		klog.Fatalf("failed to parse duration %q: %v", *certDurationString, err)
+	}
+
+	nodeName := os.Getenv("MULTUS_NODE_NAME")
+	if nodeName == "" {
+		klog.Fatalf("cannot identify node name from MULTUS_NODE_NAME env variables")
+	}
+
+	// retrieve API server from bootstrapConfig()
+	config, err := clientcmd.BuildConfigFromFlags("", *bootstrapConfig)
+	if err != nil {
+		klog.Fatalf("cannot get in-cluster config: %v", err)
+	}
+	apiServer := fmt.Sprintf("%s%s", config.Host, config.APIPath)
+	caData := base64.StdEncoding.EncodeToString(config.CAData)
+
+	// run certManager to create certification
+	if _, err = k8sclient.PerNodeK8sClient(nodeName, *bootstrapConfig, certDuration, *certDir); err != nil {
+		klog.Fatalf("failed to start cert manager: %v", err)
+	}
+
+	fp, err := os.OpenFile(kubeconfigPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		klog.Fatalf("cannot create kubeconfig file %q: %v", kubeconfigPath, err)
+	}
+
+	// render kubeconfig
+	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
+	if err != nil {
+		klog.Fatalf("template parse error: %v", err)
+	}
+	templateData := map[string]string{
+		"CADATA":        caData,
+		"CERTDIR":       *certDir,
+		"K8S_APISERVER": apiServer,
+	}
+	// genearate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		klog.Fatalf("cannot create kubeconfig: %v", err)
+	}
+	if err = fp.Close(); err != nil {
+		klog.Fatalf("cannot save kubeconfig: %v", err)
+	}
+
+	klog.Infof("kubeconfig %q is saved", kubeconfigPath)
+
+	// wait for signal
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+	klog.Infof("signal received. remove kubeconfig %q and quit.", kubeconfigPath)
+	err = os.Remove(kubeconfigPath)
+	if err != nil {
+		klog.Errorf("failed to remove kubeconfig %q: %v", kubeconfigPath, err)
+	}
+}

cmd/multus-daemon/main.go

@@ -23,10 +23,12 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/signal"
 	"os/user"
 	"path/filepath"
+	"sync"
+	"syscall"
 
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
@@ -34,6 +36,7 @@ import (
 	srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -53,96 +56,90 @@ func main() {
 		os.Exit(4)
 	}
 
-	configWatcherStopChannel := make(chan struct{})
-	configWatcherDoneChannel := make(chan struct{})
-	serverStopChannel := make(chan struct{})
-	serverDoneChannel := make(chan struct{})
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
 
 	daemonConf, err := cniServerConfig(*configFilePath)
 	if err != nil {
 		os.Exit(1)
 	}
 
-	if err := startMultusDaemon(daemonConf, serverStopChannel, serverDoneChannel); err != nil {
-		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
-		os.Exit(3)
-	}
-
 	multusConf, err := config.ParseMultusConfig(*configFilePath)
 	if err != nil {
 		logging.Panicf("startMultusDaemon failed to load the multus configuration: %v", err)
 		os.Exit(1)
 	}
 
-	// Generate multus CNI config from current CNI config
+	logging.Verbosef("multus-daemon started")
+
+	if multusConf.ReadinessIndicatorFile != "" {
+		// Check readinessindicator file before daemon launch
+		logging.Verbosef("Readiness Indicator file check")
+		if err := types.GetReadinessIndicatorFile(multusConf.ReadinessIndicatorFile); err != nil {
+			_ = logging.Errorf("have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", multusConf.ReadinessIndicatorFile, err)
+			os.Exit(1)
+		}
+		logging.Verbosef("Readiness Indicator file check done!")
+	}
+
+	var configManager *config.Manager
+	var ignoreReadinessIndicator bool
 	if multusConf.MultusConfigFile == "auto" {
 		if multusConf.CNIVersion == "" {
 			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
 		}
 
-		multusConf.SocketDir = daemonConf.SocketDir
-
-		var configManager *config.Manager
-		if multusConf.MultusMasterCni == "" {
-			configManager, err = config.NewManager(*multusConf, multusConf.MultusAutoconfigDir, multusConf.ForceCNIVersion)
-		} else {
-			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
-				*multusConf, multusConf.MultusAutoconfigDir, multusConf.MultusMasterCni, multusConf.ForceCNIVersion)
-		}
+		// Generate multus CNI config from current CNI config
+		configManager, err = config.NewManager(*multusConf)
 		if err != nil {
 			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
 			os.Exit(2)
 		}
-
-		if multusConf.OverrideNetworkName {
-			if err := configManager.OverrideNetworkName(); err != nil {
-				_ = logging.Errorf("could not override the network name: %v", err)
-			}
-		}
-
-		generatedMultusConfig, err := configManager.GenerateConfig()
-		if err != nil {
-			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
-		}
-		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
-
-		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
-			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
-		}
-
-		go func(stopChannel chan<- struct{}, doneChannel chan<- struct{}) {
-			if err := configManager.MonitorPluginConfiguration(configWatcherStopChannel, doneChannel); err != nil {
-				_ = logging.Errorf("error watching file: %v", err)
-			}
-		}(configWatcherStopChannel, configWatcherDoneChannel)
-
-		<-configWatcherDoneChannel
+		// ConfigManager watches the readiness indicator file (if configured)
+		// and exits the daemon when that is removed. The CNIServer does
+		// not need to re-do that check every CNI operation
+		ignoreReadinessIndicator = true
 	} else {
 		if err := copyUserProvidedConfig(multusConf.MultusConfigFile, multusConf.CniConfigDir); err != nil {
 			logging.Errorf("failed to copy the user provided configuration %s: %v", multusConf.MultusConfigFile, err)
 		}
 	}
 
-	serverDone := false
-	configWatcherDone := false
-	for {
-		select {
-		case <-configWatcherDoneChannel:
-			logging.Verbosef("ConfigWatcher done")
-			configWatcherDone = true
-		case <-serverDoneChannel:
-			logging.Verbosef("multus-server done.")
-			serverDone = true
-		}
+	if err := startMultusDaemon(ctx, daemonConf, ignoreReadinessIndicator); err != nil {
+		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
+		os.Exit(3)
+	}
 
-		if serverDone && configWatcherDone {
-			return
+	// Wait until daemon ready
+	logging.Verbosef("API readiness check")
+	if api.WaitUntilAPIReady(daemonConf.SocketDir) != nil {
+		logging.Panicf("failed to ready multus-daemon socket: %v", err)
+		os.Exit(1)
+	}
+	logging.Verbosef("API readiness check done!")
+
+	signalCh := make(chan os.Signal, 16)
+	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		for sig := range signalCh {
+			logging.Verbosef("caught %v, stopping...", sig)
+			cancel()
+		}
+	}()
+
+	var wg sync.WaitGroup
+	if configManager != nil {
+		if err := configManager.Start(ctx, &wg); err != nil {
+			_ = logging.Errorf("failed to start config manager: %v", err)
+			os.Exit(3)
 		}
 	}
-	// never reached
+
+	wg.Wait()
+	logging.Verbosef("multus daemon is exited")
 }
 
-func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}, done chan struct{}) error {
+func startMultusDaemon(ctx context.Context, daemonConfig *srv.ControllerNetConf, ignoreReadinessIndicator bool) error {
 	if user, err := user.Current(); err != nil || user.Uid != "0" {
 		return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
 	}
@@ -151,17 +148,17 @@ func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}
 		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
 	}
 
-	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents)
+	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents, ignoreReadinessIndicator)
 	if err != nil {
 		return fmt.Errorf("failed to create the server: %v", err)
 	}
 
 	if daemonConfig.MetricsPort != nil {
-		go utilwait.Until(func() {
+		go utilwait.UntilWithContext(ctx, func(_ context.Context) {
 			http.Handle("/metrics", promhttp.Handler())
 			logging.Debugf("metrics port: %d", *daemonConfig.MetricsPort)
 			logging.Debugf("metrics: %s", http.ListenAndServe(fmt.Sprintf(":%d", *daemonConfig.MetricsPort), nil))
-		}, 0, stopCh)
+		}, 0)
 	}
 
 	l, err := srv.GetListener(api.SocketPath(daemonConfig.SocketDir))
@@ -169,23 +166,23 @@ func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}
 		return fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(daemonConfig.SocketDir), err)
 	}
 
-	server.SetKeepAlivesEnabled(false)
+	server.Start(ctx, l)
+
 	go func() {
-		utilwait.Until(func() {
-			logging.Debugf("open for business")
-			if err := server.Serve(l); err != nil {
-				utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
-			}
-		}, 0, stopCh)
-		server.Shutdown(context.TODO())
-		close(done)
+		<-ctx.Done()
+		server.Shutdown(context.Background())
 	}()
 
 	return nil
 }
 
 func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
-	configFileContents, err := os.ReadFile(configFilePath)
+	path, err := filepath.Abs(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("illegal path %s in server config path %s: %w", path, configFilePath, err)
+	}
+
+	configFileContents, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
@@ -193,9 +190,14 @@ func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
 }
 
 func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
-	srcFile, err := os.Open(multusConfigPath)
+	path, err := filepath.Abs(multusConfigPath)
 	if err != nil {
-		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+		return fmt.Errorf("illegal path %s in multusConfigPath %s: %w", path, multusConfigPath, err)
+	}
+
+	srcFile, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", path, err)
 	}
 
 	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)

cmd/multus-shim/main.go

@@ -44,15 +44,23 @@ func main() {
 		return
 	}
 
-	skel.PluginMain(
-		func(args *skel.CmdArgs) error {
-			return api.CmdAdd(args)
-		},
-		func(args *skel.CmdArgs) error {
-			return api.CmdCheck(args)
-		},
-		func(args *skel.CmdArgs) error {
-			return api.CmdDel(args)
+	skel.PluginMainFuncs(
+		skel.CNIFuncs{
+			Add: func(args *skel.CmdArgs) error {
+				return api.CmdAdd(args)
+			},
+			Check: func(args *skel.CmdArgs) error {
+				return api.CmdCheck(args)
+			},
+			Del: func(args *skel.CmdArgs) error {
+				return api.CmdDel(args)
+			},
+			GC: func(args *skel.CmdArgs) error {
+				return api.CmdGC(args)
+			},
+			Status: func(args *skel.CmdArgs) error {
+				return api.CmdStatus(args)
+			},
 		},
 		cniversion.All, "meta-plugin that delegates to other CNI plugins")
 }

cmd/multus/main.go

@@ -43,17 +43,27 @@ func main() {
 		return
 	}
 
-	skel.PluginMain(
-		func(args *skel.CmdArgs) error {
-			result, err := multus.CmdAdd(args, nil, nil)
-			if err != nil {
-				return err
-			}
-			return result.Print()
+	skel.PluginMainFuncs(
+		skel.CNIFuncs{
+			Add: func(args *skel.CmdArgs) error {
+				result, err := multus.CmdAdd(args, nil, nil)
+				if err != nil {
+					return err
+				}
+				return result.Print()
+			},
+			Del: func(args *skel.CmdArgs) error {
+				return multus.CmdDel(args, nil, nil)
+			},
+			Check: func(args *skel.CmdArgs) error {
+				return multus.CmdCheck(args, nil, nil)
+			},
+			GC: func(args *skel.CmdArgs) error {
+				return multus.CmdGC(args, nil, nil)
+			},
+			Status: func(args *skel.CmdArgs) error {
+				return multus.CmdStatus(args, nil, nil)
+			},
 		},
-		func(args *skel.CmdArgs) error {
-			return multus.CmdCheck(args, nil, nil)
-		},
-		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
 		cniversion.All, "meta-plugin that delegates to other CNI plugins")
 }

cmd/passthru-cni/main.go

@@ -0,0 +1,58 @@
+// Package: passthru-cni
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniTypes "github.com/containernetworking/cni/pkg/types"
+	current "github.com/containernetworking/cni/pkg/types/100"
+	cniVersion "github.com/containernetworking/cni/pkg/version"
+)
+
+// NetConf is a CNI configuration structure
+type NetConf struct {
+	cniTypes.NetConf
+}
+
+func main() {
+	skel.PluginMain(
+		cmdAdd,
+		nil,
+		cmdDel,
+		cniVersion.PluginSupports("0.3.0", "0.3.1", "0.4.0", "1.0.0", "1.1.0"),
+		"Passthrough CNI Plugin v1.0",
+	)
+}
+
+func cmdAdd(args *skel.CmdArgs) error {
+	n, err := loadNetConf(args.StdinData)
+	if err != nil {
+		return fmt.Errorf("passthru cni: error parsing CNI configuration: %s", err)
+	}
+
+	// Create an empty but valid CNI result
+	result := &current.Result{
+		CNIVersion: n.CNIVersion,
+		Interfaces: []*current.Interface{},
+		IPs:        []*current.IPConfig{},
+		Routes:     []*cniTypes.Route{},
+		DNS:        cniTypes.DNS{},
+	}
+
+	return cniTypes.PrintResult(result, n.CNIVersion)
+}
+
+func cmdDel(_ *skel.CmdArgs) error {
+	// Nothing to do for DEL command, just return nil
+	return nil
+}
+
+func loadNetConf(bytes []byte) (*NetConf, error) {
+	n := &NetConf{}
+	if err := json.Unmarshal(bytes, n); err != nil {
+		return nil, fmt.Errorf("passthru cni: failed to load netconf: %s", err)
+	}
+	return n, nil
+}

cmd/thin_entrypoint/main.go

@@ -21,7 +21,6 @@ import (
 	b64 "encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -32,6 +31,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
 )
 
 // Options stores command line options
@@ -41,6 +41,7 @@ type Options struct {
 	CNIVersion               string
 	MultusConfFile           string
 	MultusBinFile            string // may be hidden or remove?
+	MultusCNIConfDir         string
 	SkipMultusBinaryCopy     bool
 	MultusKubeConfigFileHost string
 	MultusMasterCNIFileName  string
@@ -57,6 +58,7 @@ type Options struct {
 	AdditionalBinDir         string
 	ForceCNIVersion          bool
 	SkipTLSVerify            bool
+	SkipMultusConfWatch      bool
 }
 
 const (
@@ -72,6 +74,7 @@ func (o *Options) addFlags() {
 	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
 	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
 	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
+	fs.StringVar(&o.MultusCNIConfDir, "multus-cni-conf-dir", "/host/etc/cni/multus/net.d", "multus specific CNI config directory")
 	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")
 
 	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
@@ -83,7 +86,8 @@ func (o *Options) addFlags() {
 	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
 	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
 	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
+	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
 	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
@@ -138,18 +142,56 @@ contexts:
 current-context: multus-context
 `
 
-func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
-	// check file exists
-	if _, err := os.Stat(serviceAccountTokenFile); err != nil {
-		return nil, fmt.Errorf("service account token is not found: %v", err)
+func getFileAndHash(filepath string) ([]byte, []byte, error) {
+	if _, err := os.Stat(filepath); err != nil {
+		return nil, nil, fmt.Errorf("file %s not found: %v", filepath, err)
 	}
-	if _, err := os.Stat(serviceAccountCAFile); err != nil {
-		return nil, fmt.Errorf("service account ca is not found: %v", err)
+	content, err := os.ReadFile(filepath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot read %s file: %v", filepath, err)
+	}
+
+	hash := sha256.New()
+	hash.Write(content)
+	return content, hash.Sum(nil), nil
+}
+
+func (o *Options) createKubeConfig(prevCAHash, prevSATokenHash []byte) ([]byte, []byte, error) {
+	caFileByte, caHash, err := getFileAndHash(serviceAccountCAFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	saTokenByte, saTokenHash, err := getFileAndHash(serviceAccountTokenFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caUnchanged := prevCAHash != nil && bytes.Equal(prevCAHash, caHash)
+	saUnchanged := prevSATokenHash != nil && bytes.Equal(prevSATokenHash, saTokenHash)
+
+	if o.SkipTLSVerify {
+		if saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	} else {
+		if caUnchanged && saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	}
+
+	if prevSATokenHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("CA (%v) or SA token (%v) changed - recreating kubeconfig\n", !caUnchanged, !saUnchanged)
 	}
 
 	// create multus.d directory
 	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
-		return nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+		return nil, nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+	}
+
+	// create multus cni conf directory
+	if err := os.MkdirAll(o.MultusCNIConfDir, 0755); err != nil {
+		return nil, nil, fmt.Errorf("cannot create multus-cni-conf-dir(%s) directory: %v", o.MultusCNIConfDir, err)
 	}
 
 	// get Kubernetes service protocol/host/port
@@ -166,30 +208,21 @@ func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
 		tlsConfig = "insecure-skip-tls-verify: true"
 	} else {
 		// create tlsConfig by service account CA file
-		caFileByte, err := os.ReadFile(serviceAccountCAFile)
-		if err != nil {
-			return nil, fmt.Errorf("cannot read service account ca file: %v", err)
-		}
 		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
 		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
 	}
 
-	saTokenByte, err := os.ReadFile(serviceAccountTokenFile)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read service account token file: %v", err)
-	}
-
 	// create kubeconfig by template and replace it by atomic
 	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
 	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
 	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
+		return nil, nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
 	}
 
 	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
 	if err != nil {
-		return nil, fmt.Errorf("template parse error: %v", err)
+		return nil, nil, fmt.Errorf("template parse error: %v", err)
 	}
 	templateData := map[string]string{
 		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
@@ -197,38 +230,27 @@ func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
 		"KubeServiceAccountToken": string(saTokenByte),
 	}
 
-	// Prepare
-	hash := sha256.New()
-	writer := io.MultiWriter(hash, fp)
-
-	// genearate kubeconfig from template
-	if err = templateKubeconfig.Execute(writer, templateData); err != nil {
-		return nil, fmt.Errorf("cannot create kubeconfig: %v", err)
+	// generate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		return nil, nil, fmt.Errorf("cannot create kubeconfig: %v", err)
 	}
 
 	if err := fp.Sync(); err != nil {
 		os.Remove(fp.Name())
-		return nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
+		return nil, nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
 	}
 	if err := fp.Close(); err != nil {
 		os.Remove(fp.Name())
-		return nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
-	}
-
-	newFileHash := hash.Sum(nil)
-	if currentFileHash != nil && bytes.Compare(newFileHash, currentFileHash) == 0 {
-		fmt.Printf("kubeconfig is same, not copy\n")
-		os.Remove(fp.Name())
-		return currentFileHash, nil
+		return nil, nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
 	}
 
 	// replace file with tempfile
 	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
-		return nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
+		return nil, nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
 	}
 
 	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
-	return newFileHash, nil
+	return caHash, saTokenHash, nil
 }
 
 const multusConflistTemplate = `{
@@ -249,6 +271,8 @@ const multusConflistTemplate = `{
             .LogFileConfig
         }}{{
             .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
         }}{{
             .ReadinessIndicatorFileConfig
         }}
@@ -277,6 +301,8 @@ const multusConfTemplate = `{
             .LogFileConfig
         }}{{
             .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
         }}{{
             .ReadinessIndicatorFileConfig
         }}
@@ -287,27 +313,57 @@ const multusConfTemplate = `{
 }
 `
 
-func (o *Options) createMultusConfig() (string, error) {
-	// find master file from MultusAutoconfigDir
+func (o *Options) getMasterConfigPath() (string, error) {
+	// Master config file is specified
+	if o.MultusMasterCNIFileName != "" {
+		return filepath.Join(o.MultusAutoconfigDir, o.MultusMasterCNIFileName), nil
+	}
+
+	// Pick the alphabetically first config file from MultusAutoconfigDir
 	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
 	if err != nil {
 		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
 	}
 
-	masterConfigPath := files[0]
-	masterConfigBytes, err := os.ReadFile(masterConfigPath)
-	if err != nil {
-		return "", fmt.Errorf("cannot read master CNI config file %q: %v", masterConfigPath, err)
+	for _, filename := range files {
+		if !strings.HasPrefix(filepath.Base(filename), "00-multus.conf") {
+			return filename, nil
+		}
 	}
+
+	// No config file found
+	return "", fmt.Errorf("cannot find valid master CNI config in %q", o.MultusAutoconfigDir)
+}
+
+func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, []byte, error) {
+	masterConfigPath, err := o.getMasterConfigPath()
+	if err != nil {
+		return "", nil, err
+	}
+
+	masterConfigBytes, masterConfigFileHash, err := getFileAndHash(masterConfigPath)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if prevMasterConfigFileHash != nil && bytes.Equal(prevMasterConfigFileHash, masterConfigFileHash) {
+		return masterConfigPath, masterConfigFileHash, nil
+	}
+
+	if prevMasterConfigFileHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("master config changed - recreating multus config\n")
+	}
+
 	masterConfig := map[string]interface{}{}
 	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
-		return "", fmt.Errorf("cannot read master CNI config json: %v", err)
+		return "", nil, fmt.Errorf("cannot read master CNI config json: %v", err)
 	}
 
 	// check CNIVersion
 	masterCNIVersionElem, ok := masterConfig["cniVersion"]
 	if !ok {
-		return "", fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
+		return "", nil, fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
 	}
 
 	if o.ForceCNIVersion {
@@ -316,7 +372,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	} else {
 		masterCNIVersion := masterCNIVersionElem.(string)
 		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
-			return "", fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
+			return "", nil, fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
 		}
 		o.CNIVersion = masterCNIVersion
 	}
@@ -327,7 +383,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if o.OverrideNetworkName {
 		masterPluginNetworkElem, ok := masterConfig["name"]
 		if !ok {
-			return "", fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
+			return "", nil, fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
 		}
 
 		masterPluginNetworkName = masterPluginNetworkElem.(string)
@@ -341,7 +397,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if isMasterConfList {
 		masterPluginsElem, ok := masterConfig["plugins"]
 		if !ok {
-			return "", fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
+			return "", nil, fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
 		}
 		masterPlugins := masterPluginsElem.([]interface{})
 		for _, v := range masterPlugins {
@@ -368,7 +424,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if len(masterCapabilities) != 0 {
 		capabilitiesByte, err := json.Marshal(masterCapabilities)
 		if err != nil {
-			return "", fmt.Errorf("cannot get capabilities map: %v", err)
+			return "", nil, fmt.Errorf("cannot get capabilities map: %v", err)
 		}
 		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
 	}
@@ -400,7 +456,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	case "":
 		// no logLevel config, skipped
 	default:
-		return "", fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
+		return "", nil, fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
 	}
 
 	// check MultusLogFile
@@ -415,6 +471,12 @@ func (o *Options) createMultusConfig() (string, error) {
 		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
 	}
 
+	// check MultusCNIConfDir
+	multusCNIConfDirConfig := ""
+	if o.MultusCNIConfDir != "" {
+		multusCNIConfDirConfig = fmt.Sprintf("\n        \"cniConf\": %q,", o.MultusCNIConfDir)
+	}
+
 	// check ReadinessIndicatorFile
 	readinessIndicatorFileConfig := ""
 	if o.ReadinessIndicatorFile != "" {
@@ -424,28 +486,28 @@ func (o *Options) createMultusConfig() (string, error) {
 	// fill .MasterPluginJSON
 	masterPluginByte, err := json.Marshal(masterConfig)
 	if err != nil {
-		return "", fmt.Errorf("cannot encode master CNI config: %v", err)
+		return "", nil, fmt.Errorf("cannot encode master CNI config: %v", err)
 	}
 
 	// generate multus config
 	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
 	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
 	if err != nil {
-		return "", fmt.Errorf("cannot create multus cni temp file: %v", err)
+		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
 	}
 
 	// use conflist template if cniVersionConfig == "1.0.0"
 	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
 	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
 	if err != nil {
-		return "", fmt.Errorf("template parse error: %v", err)
+		return "", nil, fmt.Errorf("template parse error: %v", err)
 	}
 
 	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
 		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
 		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
 		if err != nil {
-			return "", fmt.Errorf("template parse error: %v", err)
+			return "", nil, fmt.Errorf("template parse error: %v", err)
 		}
 	}
 
@@ -459,37 +521,38 @@ func (o *Options) createMultusConfig() (string, error) {
 		"LogLevelConfig":               logLevelConfig,
 		"LogFileConfig":                logFileConfig,
 		"AdditionalBinDirConfig":       additionalBinDirConfig,
+		"MultusCNIConfDirConfig":       multusCNIConfDirConfig,
 		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
 		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
 		"MasterPluginJSON":             string(masterPluginByte),
 	}
 	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
-		return "", fmt.Errorf("cannot create multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot create multus cni config: %v", err)
 	}
 
 	if err := fp.Sync(); err != nil {
 		os.Remove(tempFileName)
-		return "", fmt.Errorf("cannot flush multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot flush multus cni config: %v", err)
 	}
 	if err := fp.Close(); err != nil {
 		os.Remove(tempFileName)
-		return "", fmt.Errorf("cannot close multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot close multus cni config: %v", err)
 	}
 
 	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
-		return "", fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
+		return "", nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
 	}
 
 	if o.RenameConfFile {
 		//masterConfigPath
 		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
 		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
-			return "", fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
+			return "", nil, fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
 		}
 		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
 	}
 
-	return masterConfigPath, nil
+	return masterConfigPath, masterConfigFileHash, nil
 }
 
 func main() {
@@ -518,10 +581,15 @@ func main() {
 		}
 	}
 
-	var kubeConfigHash []byte
+	var masterConfigHash, caHash, saTokenHash []byte
 	var masterConfigFilePath string
 	// copy user specified multus conf to CNI conf directory
 	if opt.MultusConfFile != "auto" {
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
+			return
+		}
 		confFileName := filepath.Base(opt.MultusConfFile)
 		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
 		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
@@ -530,13 +598,13 @@ func main() {
 		}
 		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
 	} else { // auto generate multus config
-		kubeConfigHash, err = opt.createKubeConfig(nil)
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
 			return
 		}
 		fmt.Printf("kubeconfig file is created.\n")
-		masterConfigFilePath, err = opt.createMultusConfig()
+		masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(nil)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
 			return
@@ -544,42 +612,72 @@ func main() {
 		fmt.Printf("multus config file is created.\n")
 	}
 
-	if opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" {
+	ctx := signals.SetupSignalHandler()
+
+	if opt.CleanupConfigOnExit {
+		defer cleanupMultusConf(&opt)
+	}
+
+	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
+	if watchChanges {
 		fmt.Printf("Entering watch loop...\n")
-		for {
-			// Check kubeconfig and update if different (i.e. service account updated)
-			kubeConfigHash, err = opt.createKubeConfig(kubeConfigHash)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
-				return
-			}
+		masterConfigExists := true
 
-			// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
-			_, err = os.Stat(masterConfigFilePath)
-
-			// if masterConfigFilePath is no longer exists
-			if os.IsNotExist(err) {
-				fmt.Printf("Master plugin @ %q has been deleted. Allowing 45 seconds for its restoration...\n", masterConfigFilePath)
-				time.Sleep(10 * time.Second)
-
-				for range time.Tick(1 * time.Second) {
-					_, err = os.Stat(masterConfigFilePath)
-					if !os.IsNotExist(err) {
-						fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
-						break
-					}
+	outer:
+		for range time.Tick(1 * time.Second) {
+			select {
+			case <-ctx.Done():
+				// signal received break from loop
+				break outer
+			default:
+				// Check kubeconfig and update if different (i.e. service account updated)
+				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
+					return
+				}
+
+				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
+				_, err = os.Stat(masterConfigFilePath)
+
+				// if masterConfigFilePath is no longer exists
+				if os.IsNotExist(err) {
+					if masterConfigExists {
+						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
+					}
+					masterConfigExists = false
+					continue
+				}
+
+				if !masterConfigExists {
+					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
+					masterConfigExists = true
+				}
+
+				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+					return
 				}
-			}
-			masterConfigFilePath, err = opt.createMultusConfig()
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
-				return
 			}
 		}
 	} else {
-		// sleep infinitely
-		for {
-			time.Sleep(time.Duration(1<<63 - 1))
-		}
+		// wait until signal received
+		<-ctx.Done()
 	}
 }
+
+func cleanupMultusConf(opt *Options) {
+	// try remove multus conf
+	if opt.MultusConfFile == "auto" {
+		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+	} else {
+		confFileName := filepath.Base(opt.MultusConfFile)
+		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
+	}
+
+}

cmd/thin_entrypoint/main_test.go

@@ -1,14 +1,39 @@
 package main
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"syscall"
 	"testing"
 
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo/v2" //nolint:golint
+	. "github.com/onsi/gomega"    //nolint:golint
 )
 
+// chrootTestHelper performs chroot syscall, returns func to get back to original root or error if occurred
+func chrootTestHelper(path string) (func() error, error) {
+	root, err := os.Open("/")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := syscall.Chroot(path); err != nil {
+		root.Close()
+		return nil, err
+	}
+
+	return func() error {
+		defer root.Close()
+		if err := root.Chdir(); err != nil {
+			return err
+		}
+		return syscall.Chroot(".")
+	}, nil
+}
+
 func TestThinEntrypoint(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "thin_entrypoint")
@@ -103,14 +128,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -151,14 +177,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -200,7 +227,7 @@ var _ = Describe("thin entrypoint testing", func() {
 		err = os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)
 		Expect(err).NotTo(HaveOccurred())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
@@ -210,11 +237,13 @@ var _ = Describe("thin entrypoint testing", func() {
 			MultusLogLevel:           "DEBUG",
 			MultusLogFile:            "/tmp/foobar.log",
 			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
 			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -225,6 +254,7 @@ var _ = Describe("thin entrypoint testing", func() {
         "logLevel": "debug",
         "logFile": "/tmp/foobar.log",
         "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
         "readinessindicatorfile": "/var/lib/foobar_indicator",
         "kubeconfig": "/etc/foobar_kubeconfig",
         "delegates": [
@@ -258,14 +288,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -308,14 +339,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -358,7 +390,7 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
@@ -368,11 +400,13 @@ var _ = Describe("thin entrypoint testing", func() {
 			MultusLogLevel:           "DEBUG",
 			MultusLogFile:            "/tmp/foobar.log",
 			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
 			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -385,6 +419,7 @@ var _ = Describe("thin entrypoint testing", func() {
         "logLevel": "debug",
         "logFile": "/tmp/foobar.log",
         "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
         "readinessindicatorfile": "/var/lib/foobar_indicator",
         "kubeconfig": "/etc/foobar_kubeconfig",
         "delegates": [
@@ -399,4 +434,111 @@ var _ = Describe("thin entrypoint testing", func() {
 		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
+	It("Run createMultusConfig(), with options, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfigFileName := "10-testcni.conf"
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, masterCNIConfigFileName), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		// create another CNI config
+		anotherCNIConfigFileName := "09-test2cni.conf" // Alphabetically before masterCNIConfigFileName
+		anotherCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test2",
+			"type": "cnitest2type"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, anotherCNIConfigFileName), []byte(anotherCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			MultusMasterCNIFileName:  masterCNIConfigFileName,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createKubeConfig()", func() {
+		// create temp dir and files
+		tmpDir := GinkgoT().TempDir()
+
+		cniConfDir := "/cni_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, cniConfDir), 0755)).To(Succeed())
+
+		multusConfDir := "/multus_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, multusConfDir), 0755)).To(Succeed())
+
+		// Create service account CA file and token file with dummy data
+		svcAccountPath := filepath.Join(tmpDir, "var/run/secrets/kubernetes.io/serviceaccount")
+		Expect(os.MkdirAll(svcAccountPath, 0755)).ToNot(HaveOccurred())
+		svcAccountCAFile := filepath.Join(tmpDir, serviceAccountCAFile)
+		svcAccountTokenFile := filepath.Join(tmpDir, serviceAccountTokenFile)
+		Expect(os.WriteFile(svcAccountCAFile, []byte("dummy-ca-content"), 0644)).To(Succeed())
+		Expect(os.WriteFile(svcAccountTokenFile, []byte("dummy-token-content"), 0644)).To(Succeed())
+
+		// Set up the Options struct
+		options := &Options{
+			CNIConfDir:       cniConfDir,
+			MultusCNIConfDir: multusConfDir,
+		}
+
+		// Run the createKubeConfig function in a chroot env
+		back, err := chrootTestHelper(tmpDir)
+		Expect(err).ToNot(HaveOccurred())
+		caHash, saTokenHash, err := options.createKubeConfig(nil, nil)
+		Expect(back()).ToNot(HaveOccurred())
+		// back to original root
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(caHash).NotTo(BeNil())
+		Expect(saTokenHash).NotTo(BeNil())
+
+		// Verify the kubeconfig file was created successfully
+		kubeConfigPath := filepath.Join(tmpDir, cniConfDir, "multus.d", "multus.kubeconfig")
+		content, err := os.ReadFile(kubeConfigPath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(content).NotTo(BeEmpty())
+
+		// Cleanup
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
 })

deployments/multus-daemonset-crio.yml

@@ -197,6 +197,7 @@ spec:
           privileged: true
           capabilities:
             add: ["SYS_ADMIN"]
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: run
           mountPath: /run

deployments/multus-daemonset-thick.yml

@@ -69,7 +69,9 @@ rules:
       - pods/status
     verbs:
       - get
+      - list
       - update
+      - watch
   - apiGroups:
       - ""
       - events.k8s.io
@@ -111,13 +113,13 @@ data:
   daemon-config.json: |
     {
         "chrootDir": "/hostroot",
-        "confDir": "/host/etc/cni/net.d",
-        "logLevel": "verbose",
-        "socketDir": "/host/run/multus/",
         "cniVersion": "0.3.1",
+        "logLevel": "verbose",
+        "logToStderr": true,
         "cniConfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
         "multusConfigFile": "auto",
-        "multusAutoconfigDir": "/host/etc/cni/net.d"
+        "socketDir": "/host/run/multus/"
     }
 ---
 apiVersion: apps/v1
@@ -163,15 +165,22 @@ spec:
               memory: "50Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
+            # multus-daemon expects that cnibin path must be identical between pod and container host.
+            # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
+            # not to any other directory, like '/opt/bin' or '/usr/bin'.
+            - name: cnibin
+              mountPath: /opt/cni/bin
             - name: host-run
               mountPath: /host/run
             - name: host-var-lib-cni-multus
               mountPath: /var/lib/cni/multus
             - name: host-var-lib-kubelet
               mountPath: /var/lib/kubelet
+              mountPropagation: HostToContainer
             - name: host-run-k8s-cni-cncf-io
               mountPath: /run/k8s.cni.cncf.io
             - name: host-run-netns
@@ -183,19 +192,27 @@ spec:
             - name: hostroot
               mountPath: /hostroot
               mountPropagation: HostToContainer
+            - mountPath: /etc/cni/multus/net.d
+              name: multus-conf-dir
+          env:
+            - name: MULTUS_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
       initContainers:
         - name: install-multus-binary
           image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
           command:
-            - "cp"
-            - "/usr/src/multus-cni/bin/multus-shim"
-            - "/host/opt/cni/bin/multus-shim"
+            - "sh"
+            - "-c"
+            - "cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim && cp /usr/src/multus-cni/bin/passthru /host/opt/cni/bin/passthru"
           resources:
             requests:
               cpu: "10m"
               memory: "15Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin
@@ -232,3 +249,6 @@ spec:
         - name: host-run-netns
           hostPath:
             path: /run/netns/
+        - name: multus-conf-dir
+          hostPath:
+            path: /etc/cni/multus/net.d

deployments/multus-daemonset.yml

@@ -194,6 +194,7 @@ spec:
             memory: "50Mi"
         securityContext:
           privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: cni
           mountPath: /host/etc/cni/net.d
@@ -214,6 +215,7 @@ spec:
               memory: "15Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin

docs/configuration.md

@@ -1,7 +1,16 @@
-## Multus-cni Configuration Reference
+# Multus-cni Configuration Reference
+
+## Introduction
+
+Aside from setting options for Multus, one of the goals of configuration is to set the configuration for your *default network*. The default network is also sometimes referred as the "primary CNI plugin", the "primary network", or a "default CNI plugin" and is the CNI plugin that is used to implement [the Kubernetes networking model](https://kubernetes.io/docs/concepts/services-networking/#the-kubernetes-network-model) in your cluster. Common examples include Flannel, Weave, Calico, Cillium, and OVN-Kubernetes, among others.
+
+Here we will refer to this as your default CNI plugin or default network.
+
+## Example configuration
 
 Following is the example of multus config file, in `/etc/cni/net.d/`.
-(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
+
+Example configuration using `clusterNetwork` (see also [using delegates](#using-delegates))
 
 ```
 {
@@ -23,65 +32,109 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     "capabilities": {
         "portMappings": true
     },    
-    "readinessindicatorfile": "",
     "namespaceIsolation": false,
-"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
-    "clusterNetwork": "defaultCRD",
-    "defaultNetworks": ["sidecarCRD", "flannel"],
+    "clusterNetwork": "/etc/cni/net.d/99-flannel.conf",
+    "defaultNetworks": ["sidecarCRD", "exampleNetwork"],
     "systemNamespaces": ["kube-system", "admin"],
     "multusNamespace": "kube-system",
-"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
+    "auxiliaryCNIChainName": "cni-chain-config",
+    allowTryDeleteOnErr: false
+}
+```
+
+## Index of configuration options
+
+This is a general index of options, however note that you must set either the `clusterNetwork` or `delegates` options, see the following sections after the index for details.
+
+* `name` (string, required): The name of the network
+* `type` (string, required): Must be set to the value of "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* [`logToStderr`](#Logging-via-STDERR) (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
+* [`logFile`](#Writing-to-a-Log-File) (string, optional): file path for log file. multus puts log in given file
+* [`logLevel`](#Logging-Level) (string, optional): logging level (values in decreasing order of verbosity: "debug", "error", "verbose", or "panic")
+* [`logOptions`](#Logging-Options) (object, optional): logging option, More detailed log configuration
+* [`namespaceIsolation`](#Namespace-Isolation) (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* [`globalNamespaces`](#Allow-specific-namespaces-to-be-used-across-namespaces-when-using-namespace-isolation): (string, optional): Used only when `namespaceIsolation` is true, allows specification of comma-delimited list of namespaces which may be referred to outside of namespace isolation.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* [`readinessindicatorfile`](#Default-Network-Readiness-Indicator): The path to a file whose existence denotes that the default network is ready
+message to next when some missing error. Defaults to false.
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` (the default value is `kube-system`)
+* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL 
+* [`auxiliaryCNIChainName`](#auxiliaryCNIChainName) (string, optional): Enable loading CNI configurations from disk as chained plugins in an auxiliary CNI chain
+
+### Using `clusterNetwork`
+
+Using the `clusterNetwork` option and the `delegates` are **mutually exclusive**. If `clusterNetwork` is set, the `delegates` field is *ignored*. 
+
+You **must** set one or the other.
+
+Therefore:
+
+* Set `clusterNetwork` and if this is set, optionally set the `defaultNetworks`.
+* OR you **must** set `delegates`.
+
+Options:
+
+* `clusterNetwork` (string, required if not using `delegates`): the default CNI plugin to be executed.
+* `defaultNetworks` ([]string, optional): Additional / secondary network attachment that is always attached to each pod. 
+
+The following values are valid for both `clusterNetwork` and `defaultNetworks` and are processed in the following order:
+
+* The name of a `NetworkAttachmentDefinition` custom resource in the namespace specified  by the `multusNamespace` configuration option
+* The `"name"` value in the contents of a CNI JSON configuration file in the CNI configuration directory, 
+  * The given name for `clusterNetwork` should match the value for `name` key in the contents of the CNI JSON file (e.g. `"name": "test"` in `my.conf` when `"clusterNetwork": "test"`)
+* A path to a directory containing CNI json configuration files. The alphabetically first file will be used.
+* Absolute file path for CNI config file
+* If none of the above are found using the value, Multus will raise an error.
+
+If for example you have `defaultNetworks` set as:
+
+```
+"defaultNetworks": ["sidecarNetwork", "exampleNetwork"],
+```
+
+In this example, the values in the expression refer to `NetworkAttachmentDefinition` custom resource names. Therefore, there must be `NetworkAttachmentDefinitions` already created with the names `sidecarNetwork` and `exampleNetwork`.
+
+This means that in addition to the cluster network, each pod would be assigned two additional networks by default, and the pod would present three interfaces, e.g. `eth0`, `net1`, and `net2`, with `net1` and `net2` being set by the above described `NetworkAttachmentDefinitions`. Additional attachments as made by setting `k8s.v1.cni.cncf.io/networks` on pods will be made in addition to those set in the `defaultNetworks` configuration option.
+
+### Using `delegates`
+
+If `clusterNetwork` is not set, you **must** use `delegates`.
+
+* `delegates` ([]map, required if not using `clusterNetwork`). List of CNI configurations to be used as your default CNI plugin(s).
+
+Example configuration using `delegates`:
+
+```
+{
+    "cniVersion": "0.3.1",
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
     "delegates": [{
         "type": "weave-net",
         "hairpinMode": true
     }, {
         "type": "macvlan",
         ... (snip)
-    }],
-    allowTryDeleteOnErr: false
+    }]
 }
 ```
 
-* `name` (string, required): the name of the network
-* `type` (string, required): "multus"
-* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
-* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
-* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
-* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
-* `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
-* `logFile` (string, optional): file path for log file. multus puts log in given file
-* `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
-* `logOptions` (object, optional): logging option, More detailed log configuration
-* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
-* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
-* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
-
-User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
-
-* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
-* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
-* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
-* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
-* `delegates` ([]map,required): number of delegate details in the Multus
-* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL message to next when some missing error. Defaults to false.
-
-### Network selection flow of clusterNetwork/defaultNetworks
-
-Multus will find network for clusterNetwork/defaultNetworks as following sequences:
-
-1. CRD object for given network name, in 'kube-system' namespace
-1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
-1. Directory for CNI json config file. Multus will find alphabetically first file for the network
-1. File path for CNI json confile file.
-1. Multus failed to find network. Multus raise error message
-
-## Miscellaneous config
+## Configuration Option Details
 
 ### Default Network Readiness Indicator
 
-You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
+You may desire that your default network becomes ready before attaching networks with Multus. This is disabled by default and not used unless you set the `readinessindicatorfile` option to a non-blank value.
 
-For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, therefore, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
 
 In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
 
@@ -329,3 +382,47 @@ annotations:
  v1.multus-cni.io/default-network: calico-conf
 ...
 ```
+
+### `auxiliaryCNIChainName`
+
+`auxiliaryCNIChainName` (of value string) is used to express the name of an additional auxiliary CNI chain that will execute in order to composably execute chained CNI plugins from configurations on the host's disk in a subdirectory of the CNI configuration directory.
+
+**NOTE**: The path used to determine the base for the subdirectory is the pathname of the `clusterNetwork` value, which must be set to a file in order to use this functionality.
+
+When this string is set, Multus will execute an additional CNI chain, outside of the default network, on its own independent CNI chain (as to not interfere with default network functionality that might be hampered by CNI chaining and to otherwise isolate this execution) and will load CNI configurations from a subdirectory of the same name in the CNI configuration directory.
+
+This feature is based on [improvements made to libcni for "safe subdirectory-based plugin conf loading"](https://github.com/containernetworking/cni/pull/1052). 
+
+`auxiliaryCNIChainName` is meant to be set as a CNI configuration name, this name is arbitrary but must match the subdirectory name.
+
+Consider this [daemon configuration](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/deployments/multus-daemonset-thick.yml#L113):
+
+```
+{
+  "cniConfigDir": "/host/etc/cni/net.d",
+  "multusAutoconfigDir": "/host/etc/cni/net.d",
+  "multusConfigFile": "auto",
+  "socketDir": "/host/run/multus/",
+  "auxiliaryCNIChainName": "cni-chain-config"
+}
+```
+
+Here we have set `"auxiliaryCNIChainName": "cni-chain-config"`, and we have expressed that our CNI configurations are on `/etc/cni/net.d/` on the host.
+
+In this case, we would also have a directory named in `/etc/cni/net.d/cni-chain-config`
+
+One could add any number of CNI configurations to be used as part of this chain, consider this example if we added a tuning CNI configuration called `/etc/cni/net.d/cni-chain-config/mytuning.conf` with these contents:
+
+```
+{
+  "name": "mytuning",
+  "type": "tuning",
+  "sysctl": {
+    "net.ipv4.conf.IFNAME.arp_filter": "1"
+  }
+}
+```
+
+With the given configuration, plus this configuration, this would be executed for every pod launched by Multus CNI.
+
+If this is unset, no auxiliary chain will be executed. However, if the default network CNI configuration is loaded from disk and is a conflist format, the libcni functionality for loading from a subdirectory will still apply.

docs/development.md

@@ -39,7 +39,7 @@ cd multus-cni
 ./hack/build-go.sh
 ```
 
-## How do I run CI tests?
+## How do I run the unit tests?
 
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
@@ -47,6 +47,10 @@ Multus has go unit tests (based on ginkgo framework).The following commands driv
 sudo ./hack/test-go.sh
 ```
 
+## How do I run the e2e tests?
+
+Check the `README.md` in the `./e2e/` folder.
+
 ## What are the best practices for logging?
 
 The following are the best practices for multus logging:
@@ -59,3 +63,7 @@ The following are the best practices for multus logging:
 ## Multus release schedule
 
 On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.
+
+## Multi-arch builds
+
+Multus is currently built for a number of architectures, however, our testing and validation is only performed against x86 architectures. Our x86 architecture has end to end testing, however, for other architectures, only supported via best effort community contributions.

docs/how-to-use.md

@@ -19,13 +19,13 @@ You may acquire the Multus binary via compilation (see the [developer guide](dev
 
 *Via Daemonset method*
 
-As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
+As a [quickstart](quickstart.md), you may apply these YAML files. Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    cat ./deployments/multus-daemonset.yml | kubectl apply -f -  # thin deployment
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml  # thin deployment
 
 or
 
-    cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f - # thick (client/server) deployment
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml # thick (client/server) deployment
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -126,7 +126,7 @@ Create kubeconfig at master node as following commands:
 mkdir -p /etc/cni/net.d/multus.d
 SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
 SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_PROTOCOL=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
 KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
 KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
 cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
@@ -511,7 +511,7 @@ spec:
 EOF
 ```
 
-We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation. 
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation.
 
 ```
 cat <<EOF | kubectl create -f -
@@ -537,9 +537,9 @@ This will set `192.168.2.1` as the default route over the `net1` interface, such
 ```
 kubectl exec -it samplepod -- ip route
 
-default via 192.168.2.1 dev net1 
-10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
-10.244.0.0/16 via 10.244.0.1 dev eth0 
+default via 192.168.2.1 dev net1
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169
+10.244.0.0/16 via 10.244.0.1 dev eth0
 ```
 
 ## Entrypoint Parameters
@@ -551,7 +551,7 @@ Typically, you'd modified the daemonset YAML itself to specify these parameters.
 For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
 
 ```
-  command: ["/entrypoint.sh"]
+  command: ["/thin_entrypoint"]
   args:
   - "--multus-conf-file=auto"
   - "--namespace-isolation=true"
@@ -590,7 +590,7 @@ The `--multus-conf-file` is one of two options; it can be set to a source file t
 
 The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
 
-In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will take the alphabetically first configuration there and wrap that into a Multus configuration.
 
     --multus-autoconfig-dir=/host/etc/cni/net.d
 
@@ -619,6 +619,14 @@ In some cases, the original CNI configuration that the Multus configuration was
 
     --cleanup-config-on-exit=true
 
+When specifying `--cleanup-config-on-exit=true` the entrypoint script will delete any generated/copied Multus configuration files when entrypoint script
+exits (upon Pod termination). This allows Multus to be safely removed from the cluster when its no longer needed.
+
+In addition, when both `--cleanup-config-on-exit=true` and `--multus-conf-file=auto` are specified, the entrypoint script will watch for changes of the
+master CNI configuration and kubeconfig. when such change detected, the script will re-genereate Multus configuration. Watch can be skipped by setting:
+
+    --skip-config-watch
+
 Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
 
     --rename-conf-file=true
@@ -634,3 +642,126 @@ Sometimes, you may wish to not have the entrypoint copy the binary file onto the
 If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
 
     --readiness-indicator-file=/path/to/file
+
+### Run pod with network annotation and Dynamic Resource Allocation driver
+
+> :warning: Dynamic Resource Allocation (DRA) is [currently an alpha](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/),
+> and is subject to change. Please consider this functionality as a preview. The architecture and usage of DRA in
+> Multus CNI may change in the future as this technology matures.
+>
+> The current DRA integration is based on the DRA API for Kubernetes 1.26 to 1.30. With Kubernetes 1.31, the DRA API
+> will change and multus doesn't integrate with the new API yet.
+
+Dynamic Resource Allocation is alternative mechanism to device plugin which allows to requests pod and container
+resources.
+
+The following sections describe how to use DRA with multus and NVIDIA DRA driver. Other DRA networking driver vendors
+should follow similar concepts to make use of multus DRA support.
+
+#### Prerequisite
+
+1. Kubernetes 1.27
+2. Container Runtime with CDI support enabled
+3. Kubernetes runtime-config=resource.k8s.io/v1alpha2
+4. Kubernetes feature-gates=DynamicResourceAllocation=True,KubeletPodResourcesDynamicResources=true
+
+#### Install DRA driver
+
+The current example uses NVIDIA DRA driver for networking. This DRA driver is not publicly available. An alternative to
+this DRA driver is available at [dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver).
+
+#### Create dynamic resource class with NVIDIA network DRA driver
+
+The `ResourceClass` defines the resource pool of `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClass
+metadata:
+  name: sf-pool-1
+driverName: net.resource.nvidia.com
+EOF
+```
+
+#### Create network attachment definition with resource name
+
+The `k8s.v1.cni.cncf.io/resourceName` should match the `ResourceClass` name defined in the section above.
+In this example it is `sf-pool-1`. Multus query the K8s PodResource API to fetch the `resourceClass` name and also
+query the NetworkAttachmentDefinition `k8s.v1.cni.cncf.io/resourceName`. If both has the same name multus send the
+CDI device name in the DeviceID argument.
+
+##### NetworkAttachmentDefinition for ovn-kubernetes example:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: sf-pool-1
+spec:
+  config: '{
+      "cniVersion": "0.4.0",
+      "dns": {},
+      "ipam": {},
+      "logFile": "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+      "logLevel": "4",
+      "logfile-maxage": 5,
+      "logfile-maxbackups": 5,
+      "logfile-maxsize": 100,
+      "name": "ovn-kubernetes",
+      "type": "ovn-k8s-cni-overlay"
+    }'
+EOF
+```
+
+#### Create DRA Resource Claim
+
+Following command creates `ResourceClaim` `sf` which request resource from  `ResourceClass` `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaim
+metadata:
+  namespace: default
+  name: sf
+spec:
+  spec:
+    resourceClassName: sf-pool-1
+EOF
+```
+
+#### Launch pod with DRA Resource Claim
+
+Following command Launch a Pod with primiry network `default` and `ResourceClaim` `sf`.
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: default
+  name: test-sf-claim
+  annotations:
+    v1.multus-cni.io/default-network: default
+spec:
+  restartPolicy: Always
+  containers:
+  - name: with-resource
+    image: docker.io/library/ubuntu:22.04
+    command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+    resources:
+      claims:
+      - name: resource
+  resourceClaims:
+  - name: resource
+    source:
+      resourceClaimName: sf
+```

docs/quickstart.md

@@ -42,25 +42,19 @@ master-2              Ready    master          1h     v1.17.1
 
 Our recommended quickstart method to deploy Multus is to deploy using a Daemonset (a method of running pods on each nodes in your cluster), this spins up pods which install a Multus binary and configure Multus for usage.
 
-Firstly, clone this GitHub repository. 
-
-```
-git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
-```
-
 We'll apply a YAML file with `kubectl` from this repo, which installs the Multus components.
 
 Recommended installation:
 
 ```
-cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
 See the [thick plugin docs](./thick-plugin.md) for more information about this architecture.
 
 Alternatively, you may install the thin-plugin with:
 
 ```
-cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 ```
 
 ### What the Multus daemonset does

docs/thick-plugin.md

@@ -72,6 +72,7 @@ is provided.
 - `"logLevel"`: the logging level for the multus daemon logs.
 - `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
 as well. By default, it is disabled.
+- `"auxiliaryCNIChainName"`: set a value to execute chained cni configurations from disk in an auxiliary CNI chain (see details in [configuration.md](configuration.md))
 
 In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
 

e2e/README.md

@@ -5,7 +5,7 @@
 To run the e2e test, you need the following components:
 
 - curl
-- j2cli
+- jinjanator (optional)
 - docker
 
 ### How to test e2e
@@ -14,7 +14,23 @@ To run the e2e test, you need the following components:
 $ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 $ cd multus-cni/e2e
 $ ./get_tools.sh
+```
+
+If you have `jinjanator` you can generate the YAML with:
+
+```
 $ ./generate_yamls.sh
+```
+
+Alternatively, if you have trouble with it, use the `sed` script.
+
+```
+$ ./e2e/sed_generate_yaml.sh
+```
+
+Then, setup the cluster
+
+```
 $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```

e2e/get_tools.sh

@@ -5,7 +5,7 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.27.0/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl
@@ -13,3 +13,4 @@ curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq
+wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm

e2e/sed_generate_yaml.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ ! -d yamls ]; then
+	mkdir yamls
+fi
+
+# specify CNI version (default: 0.4.0)
+CNI_VERSION=${CNI_VERSION:-0.4.0}
+
+templates_dir="$(dirname $(readlink -f $0))/templates"
+
+# generate yaml files based on templates/*.j2 to yamls directory
+for i in `ls ${templates_dir}/*.j2`; do
+	echo "Processing $i..."
+	# Use sed to replace the placeholder with the CNI_VERSION variable
+	sed "s/{{ CNI_VERSION }}/$CNI_VERSION/g" $i > yamls/$(basename ${i%.j2})
+done

e2e/setup_cluster.sh

@@ -11,31 +11,20 @@ OCI_BIN="${OCI_BIN:-docker}"
 # Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
 # Defaults to `multus-daemonset-thick.yml`.
 MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
+# define the dockerfile to build multus.
+# Acceptable values are `Dockerfile`. `Dockerfile.thick`.
+# Defaults to `Dockerfile.thick`.
+MULTUS_DOCKERFILE="${MULTUS_DOCKERFILE:-Dockerfile.thick}"
 
 kind_network='kind'
-reg_name='kind-registry'
-reg_port='5000'
-running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
-if [ "${running}" != 'true' ]; then
-  # run registry and push the multus image
-  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
-  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
-  $OCI_BIN push localhost:5000/multus:e2e
+if [ "${MULTUS_DOCKERFILE}" != "none" ]; then
+	$OCI_BIN build -t localhost:5000/multus:e2e -f ../images/${MULTUS_DOCKERFILE} ..
 fi
-reg_host="${reg_name}"
-if [ "${kind_network}" = "bridge" ]; then
-    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
-fi
-echo "Registry Host: ${reg_host}"
 
 # deploy cluster with kind
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-containerdConfigPatches:
-- |-
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_host}:${reg_port}"]
 nodes:
   - role: control-plane
   - role: worker
@@ -45,31 +34,27 @@ nodes:
       nodeRegistration:
         kubeletExtraArgs:
           pod-manifest-path: "/etc/kubernetes/manifests/"
+          feature-gates: "DynamicResourceAllocation=true,DRAResourceClaimDeviceStatus=true,KubeletPodResourcesDynamicResources=true"
   - role: worker
+# Required by DRA Integration
+##
+featureGates:
+  DynamicResourceAllocation: true
+  DRAResourceClaimDeviceStatus: true
+  KubeletPodResourcesDynamicResources: true
+runtimeConfig:
+  "api/beta": "true"
+containerdConfigPatches:
+# Enable CDI as described in
+# https://github.com/container-orchestrated-devices/container-device-interface#containerd-configuration
+- |-
+  [plugins."io.containerd.grpc.v1.cri"]
+      enable_cdi = true
+##
 EOF
 
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: local-registry-hosting
-  namespace: kube-public
-data:
-  localRegistryHosting.v1: |
-    host: "localhost:${reg_port}"
-    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
-EOF
-
-containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
-needs_connect="true"
-for c in $containers; do
-  if [ "$c" = "${reg_name}" ]; then
-    needs_connect="false"
-  fi
-done
-if [ "${needs_connect}" = "true" ]; then
-  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
-fi
+# load multus image from container host to kind node
+kind load docker-image localhost:5000/multus:e2e
 
 worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
 worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)

e2e/teardown.sh

@@ -1,10 +1,7 @@
 #!/bin/sh
 #set -o errexit
 
-reg_name='kind-registry'
 export PATH=${PATH}:./bin
 
 # delete cluster kind
 kind delete cluster
-docker kill ${reg_name}
-docker rm ${reg_name}

e2e/templates/cni-install.yml.j2

@@ -7,9 +7,9 @@ metadata:
 data:
   install_cni.sh: |
     cd /tmp
-    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
+    wget https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz
     cd /host/opt/cni/bin
-    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.4.0.tgz
     sleep infinite
 ---
 apiVersion: apps/v1

e2e/templates/dra-integration.yml.j2

@@ -0,0 +1,49 @@
+---
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaimTemplate
+metadata:
+  name: gpu.example.com
+spec:
+  spec:
+    resourceClassName: gpu.example.com
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: dra-net 
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: gpu.example.com
+spec:
+  config: '{
+        "cniVersion": "{{ CNI_VERSION }}",
+        "plugins": [{
+            "name": "mynet",
+            "type": "dummy",
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.1.2.0/24"
+            }
+        }]
+    }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dra-integration
+  labels:
+    app: dra-integration
+  annotations:
+    k8s.v1.cni.cncf.io/networks: default/dra-net
+spec:
+  containers:
+  - name: ctr0
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args: ["export; sleep 9999"]
+    resources:
+      claims:
+      - name: gpu
+  resourceClaims:
+  - name: gpu
+    source:
+      resourceClaimTemplateName: gpu.example.com

e2e/templates/multus-daemonset-thick.yml.j2

@@ -43,7 +43,9 @@ rules:
       - pods/status
     verbs:
       - get
+      - list
       - update
+      - watch
   - apiGroups:
       - ""
       - events.k8s.io
@@ -131,7 +133,6 @@ spec:
       containers:
       - name: kube-multus
         image: localhost:5000/multus:e2e
-        imagePullPolicy: Always
         command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
         resources:
           requests:
@@ -157,13 +158,21 @@ spec:
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
           readOnly: true
+        - name: kubelet-pod-resources
+          mountPath: /var/lib/kubelet/pod-resources
+          readOnly: true
+        env:
+        - name: MULTUS_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       initContainers:
       - name: install-multus-shim
         image: localhost:5000/multus:e2e
         command:
-          - "cp"
-          - "/usr/src/multus-cni/bin/multus-shim"
-          - "/host/opt/cni/bin/multus-shim"
+          - "sh"
+          - "-c"
+          - "cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim && cp /usr/src/multus-cni/bin/passthru /host/opt/cni/bin/passthru"
         resources:
           requests:
             cpu: "10m"
@@ -181,6 +190,9 @@ spec:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
+        - name: kubelet-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
         - name: multus-daemon-config
           configMap:
             name: multus-daemon-config

e2e/templates/subdirectory-chain-passthru-configupdate.yml.j2

@@ -0,0 +1,26 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "debug",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/host/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus",
+        "cniVersion": "{{ CNI_VERSION }}",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "forceCNIVersion": true,
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "auxiliaryCNIChainName": "vendor-cni-chain"
+    }

e2e/templates/subdirectory-chaining-passthru.yml.j2

@@ -0,0 +1,94 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cni-setup-script
+  namespace: default
+data:
+  setup.sh: |
+    #!/bin/bash
+    set -euxo pipefail
+
+    DEFAULT_NETWORK_CNI_NAME="vendor-cni-chain"
+
+    cleanup() {
+      echo "Cleaning up..."
+      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+      if [ $? -ne 0 ]; then
+        echo "Failed to remove sysctltwiddle.conf" >&2
+        exit 1
+      fi
+      echo "Cleanup completed successfully"
+    }
+    trap cleanup EXIT
+
+    # Create the chained CNI directory if it doesn't exist
+    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
+    if [ $? -ne 0 ]; then
+      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
+      exit 1
+    fi
+
+    # Write the chained tuning CNI config
+    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+    {
+      "cniVersion": "{{ CNI_VERSION }}",
+      "name": "sysctltwiddle",
+      "type": "tuning",
+      "sysctl": {
+        "net.ipv4.conf.eth0.arp_filter": "1"
+      }
+    }
+    EOF
+
+    if [ $? -ne 0 ]; then
+      echo "Failed to create chained CNI config" >&2
+      exit 1
+    fi
+
+    echo "CNI chained setup completed successfully."
+    sleep infinity
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cni-setup-daemonset
+  namespace: default
+  labels:
+    app: cni-setup
+spec:
+  selector:
+    matchLabels:
+      app: cni-setup
+  template:
+    metadata:
+      labels:
+        app: cni-setup
+    spec:
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      containers:
+      - name: setup
+        image: quay.io/fedora/fedora:40
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-config
+          mountPath: /host/etc/cni/net.d
+        - name: script-volume
+          mountPath: /scripts
+        command: ["/bin/bash", "/scripts/setup.sh"]
+      volumes:
+      - name: cni-config
+        hostPath:
+          path: /etc/cni/net.d
+          type: Directory
+      - name: script-volume
+        configMap:
+          name: cni-setup-script
+          items:
+          - key: setup.sh
+            path: setup.sh

e2e/templates/subdirectory-chaining-pod.yml.j2

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sysctl-modified
+spec:
+  containers:
+  - name: sysctl
+    image: quay.io/dosmith/fedora-procps
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    securityContext:
+      privileged: true

e2e/templates/subdirectory-chaining.yml.j2

@@ -0,0 +1,95 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cni-setup-script
+  namespace: default
+data:
+  setup.sh: |
+    #!/bin/bash
+    set -euxo pipefail
+
+    DEFAULT_NETWORK_CNI_NAME="kindnet"
+
+    cleanup() {
+      echo "Cleaning up..."
+      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+      if [ $? -ne 0 ]; then
+        echo "Failed to remove sysctltwiddle.conf" >&2
+        exit 1
+      fi
+      echo "Cleanup completed successfully"
+    }
+    trap cleanup EXIT
+
+    # Create the chained CNI directory if it doesn't exist
+    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
+    if [ $? -ne 0 ]; then
+      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
+      exit 1
+    fi
+
+    # Write the chained tuning CNI config
+    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+    {
+      "cniVersion": "{{ CNI_VERSION }}",
+      "name": "sysctltwiddle",
+      "type": "tuning",
+      "sysctl": {
+        "net.ipv4.conf.IFNAME.arp_filter": "1"
+      }
+    }
+    EOF
+
+    if [ $? -ne 0 ]; then
+      echo "Failed to create chained CNI config" >&2
+      exit 1
+    fi
+
+    echo "CNI chained setup completed successfully."
+    sleep infinity
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cni-setup-daemonset
+  namespace: default
+  labels:
+    app: cni-setup
+spec:
+  selector:
+    matchLabels:
+      app: cni-setup
+  template:
+    metadata:
+      labels:
+        app: cni-setup
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      containers:
+      - name: setup
+        image: quay.io/fedora/fedora:40
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-config
+          mountPath: /host/etc/cni/net.d
+        - name: script-volume
+          mountPath: /scripts
+        command: ["/bin/bash", "/scripts/setup.sh"]
+      volumes:
+      - name: cni-config
+        hostPath:
+          path: /etc/cni/net.d
+          type: Directory
+      - name: script-volume
+        configMap:
+          name: cni-setup-script
+          items:
+          - key: setup.sh
+            path: setup.sh

e2e/test-dra-integration.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# This test is using an example implementation of a DRA driver. This driver is mocking GPU resources. At our test we
+# don't care about what these resources are. We want to ensure that such resource is correctly passed in the Pod using
+# Multus configurations. A couple of notes:
+# - We explitictly pin the revision of the dra-example-driver to the branch `classic-dra` to indicate that the
+#   integration continues to work even when the dra-example-driver is updated. We know that classic-dra is supported
+#   in Kubernetes versions 1.26 to 1.30. Multus supports DRA in the aforementioned Kubernetes versions.
+# - The chart and latest is image is not published somewhere, therefore we have to build locally. This leads to slower
+#   e2e suite runs.
+echo "installing dra-example-driver"
+repo_path="repos/dra-example-driver"
+
+rm -rf $repo_path || true
+git clone --branch classic-dra https://github.com/kubernetes-sigs/dra-example-driver.git ${repo_path}
+${repo_path}/demo/build-driver.sh
+KIND_CLUSTER_NAME=kind ${repo_path}/demo/scripts/load-driver-image-into-kind.sh
+chart_path=${repo_path}/deployments/helm/dra-example-driver/
+overriden_values_path=${chart_path}/overriden_values.yaml
+
+# With the thick plugin, in kind, the primary network on the control plane is not always working as expected. The pods
+# sometimes are not able to communicate with the control plane and the error looks like this:
+# failed to list *v1alpha2.PodSchedulingContext: Get "https://10.96.0.1:443/apis/resource.k8s.io/v1alpha2/podschedulingcontexts?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
+# We override the values here to schedule the controller on the worker nodes where the network is working as expected.
+cat <<EOF >> ${overriden_values_path}
+controller:
+  nodeSelector: null
+  tolerations: null
+EOF
+
+helm install \
+    -n dra-example-driver \
+    --create-namespace \
+    -f ${overriden_values_path} \
+    dra-example-driver \
+    ${chart_path}
+
+echo "installing testing pods"
+kubectl create -f yamls/dra-integration.yml
+kubectl wait --for=condition=ready -l app=dra-integration --timeout=300s pod
+
+echo "check dra-integration pod for DRA injected environment variable"
+
+# We can validate that the resource is correctly injected by checking an environment variable this dra driver is injecting
+# in the Pod.
+# https://github.com/kubernetes-sigs/dra-example-driver/blob/be2b8b1db47b8c757440e955ce5ced88c23bfe86/cmd/dra-example-kubeletplugin/cdi.go#L71C20-L71C44
+env_variable=$(kubectl exec dra-integration -- bash -c "echo \$DRA_RESOURCE_DRIVER_NAME | grep gpu.resource.example.com")
+if [ $? -eq 0 ];then
+	echo "dra-integration pod has DRA injected environment variable"
+else
+	echo "dra-integration pod doesn't have DRA injected environment variable"
+	exit 1
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/dra-integration.yml
+helm uninstall -n dra-example-driver dra-example-driver

e2e/test-subdirectory-chaining-passthru.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${PATH}:./bin
+
+TEST_POD_NAME="sysctl-modified"
+EXPECTED_BINARIES="${EXPECTED_BINARIES:-/opt/cni/bin/ptp /opt/cni/bin/portmap /opt/cni/bin/tuning}"
+EXPECTED_CNI_DIR="/etc/cni/net.d"
+
+# Reconfigure multus
+echo "Applying subdirectory chain passthru config..."
+kubectl apply -f yamls/subdirectory-chain-passthru-configupdate.yml
+
+# Restart the multus daemonset to pick up the new config
+echo "Restarting Multus DaemonSet..."
+kubectl rollout restart daemonset kube-multus-ds-amd64 -n kube-system
+kubectl rollout status daemonset/kube-multus-ds-amd64 -n kube-system
+
+# Debug: show CNI configs and binaries inside each Kind node
+echo "Checking CNI configs and binaries on nodes..."
+
+for node in $(kubectl get nodes --no-headers | awk '{print $1}'); do
+    container_name=$(docker ps --format '{{.Names}}' | grep "^${node}$")
+
+    echo "------"
+    echo "Node: ${node} (container: ${container_name})"
+    echo "Listing /opt/cni/bin contents..."
+    docker exec "${container_name}" ls -l /opt/cni/bin || echo "WARNING: /opt/cni/bin missing!"
+
+    echo "Checking expected binaries..."
+    for bin in $EXPECTED_BINARIES; do
+        echo "Checking for ${bin}..."
+        if docker exec "${container_name}" test -f "${bin}"; then
+            echo "SUCCESS: ${bin} found."
+        else
+            echo "FAIL: ${bin} NOT found!"
+        fi
+    done
+
+    echo "Listing /etc/cni/net.d configs..."
+    docker exec "${container_name}" ls -l ${EXPECTED_CNI_DIR} || echo "WARNING: ${EXPECTED_CNI_DIR} missing!"
+done
+echo "------"
+
+# Deploy the daemonset that will lay down the chained CNI config
+echo "Applying CNI setup DaemonSet..."
+kubectl apply -f yamls/subdirectory-chaining-passthru.yml
+
+# Wait for the daemonset pods to be ready (make sure they set up CNI config)
+echo "Waiting for CNI setup DaemonSet to be Ready..."
+kubectl rollout status daemonset/cni-setup-daemonset --timeout=300s
+
+# Deploy a test pod that will get chained CNI applied
+echo "Applying test pod..."
+kubectl apply -f yamls/subdirectory-chaining-pod.yml
+
+# Wait for the pod to be Ready
+echo "Waiting for test pod to be Ready..."
+kubectl wait --for=condition=ready pod/${TEST_POD_NAME} --timeout=300s
+
+# Check that the sysctl got set
+echo "Verifying sysctl arp_filter is set to 1 on eth0..."
+
+SYSCTL_VALUE=$(kubectl exec ${TEST_POD_NAME} -- sysctl -n net.ipv4.conf.eth0.arp_filter)
+
+if [ "$SYSCTL_VALUE" != "1" ]; then
+  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
+  exit 1
+else
+  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
+fi
+
+# Cleanup
+echo "Cleaning up test resources..."
+kubectl delete -f yamls/subdirectory-chaining-pod.yml
+kubectl delete -f yamls/subdirectory-chaining-passthru.yml
+
+echo "Test completed successfully."
+exit 0

e2e/test-subdirectory-chaining.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+TEST_POD_NAME="sysctl-modified"
+
+# Deploy the daemonset that will lay down the chained CNI config
+kubectl apply -f yamls/subdirectory-chaining.yml
+
+# Wait for the daemonset pods to be ready (we need the config to be laid down)
+kubectl rollout status daemonset/cni-setup-daemonset
+
+# Deploy a test pod that will get chained CNI applied
+kubectl apply -f yamls/subdirectory-chaining-pod.yml
+
+# Wait for the pod to be Ready
+kubectl wait --for=condition=ready pod/sysctl-modified --timeout=300s
+
+# Check that the sysctl got set properly inside the pod's eth0 interface
+echo "Verifying sysctl arp_filter is set to 1 on eth0"
+
+SYSCTL_VALUE=$(kubectl exec sysctl-modified -- sysctl -n net.ipv4.conf.eth0.arp_filter)
+
+if [ "$SYSCTL_VALUE" != "1" ]; then
+  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
+  exit 1
+else
+  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
+fi
+
+# 6. Clean up
+echo "Cleaning up test resources"
+kubectl delete -f yamls/subdirectory-chaining-pod.yml
+kubectl delete -f yamls/subdirectory-chaining.yml
+
+exit 0

examples/macvlan-pod.yml

@@ -54,3 +54,4 @@ spec:
     image: dougbtv/centos-network
     ports:
     - containerPort: 80
+  automountServiceAccountToken: false

examples/sriov-pod.yml

@@ -45,3 +45,4 @@ spec:
       limits:
         intel.com/sriov: '1'
   restartPolicy: "Never"
+  automountServiceAccountToken: false

go.mod

@@ -1,96 +1,78 @@
 module gopkg.in/k8snetworkplumbingwg/multus-cni.v4
 
-go 1.18
+go 1.21
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/containernetworking/cni v1.1.2
+	github.com/containernetworking/cni v1.3.0
 	github.com/containernetworking/plugins v1.1.0
 	github.com/fsnotify/fsnotify v1.6.0
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
-	github.com/onsi/ginkgo/v2 v2.5.1
-	github.com/onsi/gomega v1.24.0
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6
+	github.com/onsi/ginkgo/v2 v2.20.1
+	github.com/onsi/gomega v1.34.1
+	github.com/prometheus/client_golang v1.16.0
+	github.com/spf13/pflag v1.0.5
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
-	golang.org/x/net v0.7.0
-	golang.org/x/sys v0.5.0
-	google.golang.org/grpc v1.40.0
+	golang.org/x/net v0.28.0
+	golang.org/x/sys v0.23.0
+	google.golang.org/grpc v1.58.3
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	k8s.io/api v0.22.8
-	k8s.io/apimachinery v0.22.8
-	k8s.io/client-go v0.22.8
+	k8s.io/api v0.29.0
+	k8s.io/apimachinery v0.29.0
+	k8s.io/client-go v0.29.0
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.60.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 // indirect
-	k8s.io/kubelet v0.22.8
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	k8s.io/klog/v2 v2.110.1
+	k8s.io/kubelet v0.27.5
 )
 
-require github.com/prometheus/client_golang v1.12.2
-
 require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/term v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-)
-
-replace (
-	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
-	k8s.io/api => k8s.io/api v0.22.8
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.8
-	k8s.io/apimachinery => k8s.io/apimachinery v0.22.8
-	k8s.io/apiserver => k8s.io/apiserver v0.22.8
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.22.8
-	k8s.io/client-go => k8s.io/client-go v0.22.8
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.22.8
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.8
-	k8s.io/code-generator => k8s.io/code-generator v0.22.8
-	k8s.io/component-base => k8s.io/component-base v0.22.8
-	k8s.io/component-helpers => k8s.io/component-helpers v0.22.8
-	k8s.io/controller-manager => k8s.io/controller-manager v0.22.8
-	k8s.io/cri-api => k8s.io/cri-api v0.22.8
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.22.8
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.22.8
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.22.8
-	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.22.8
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.22.8
-	k8s.io/kubectl => k8s.io/kubectl v0.22.8
-	k8s.io/kubelet => k8s.io/kubelet v0.22.8
-	k8s.io/kubernetes => k8s.io/kubernetes v1.22.8
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.22.8
-	k8s.io/metrics => k8s.io/metrics v0.22.8
-	k8s.io/mount-utils => k8s.io/mount-utils v0.22.8
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.8
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.22.8
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -1,810 +1,219 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
-github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo=
+github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4=
 github.com/containernetworking/plugins v1.1.0 h1:kTIldaDo9SlbQsjhUKvDx0v9q7zyIFJH/Rm9F4xRBro=
 github.com/containernetworking/plugins v1.1.0/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU=
+github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 h1:lhSaboKtal0XF2yqSw2BqNB1vUL4+a4BFe39I9G/yiM=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw=
-github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg=
-github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo=
+github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
+github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
+github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA=
 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
-github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
-go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
-go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
-go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
-go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
-go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
-go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
-go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
-go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
+golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.22.8 h1:7Ld6tHuvaYzcQE2axLmomWlhP0fK3vpLfo6fBaNrCIs=
-k8s.io/api v0.22.8/go.mod h1:uLlWJNRJ+AYwgAdsNwf0TsD3eByNYW9RlXFmkMdL3yk=
-k8s.io/apimachinery v0.22.8 h1:kazMo4/t5ZPI7MwImnCJODZrt1VuwbYBixhTzaNIxsw=
-k8s.io/apimachinery v0.22.8/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU=
-k8s.io/client-go v0.22.8 h1:dWgwPqpWH/DPLWSczA6b61VxFIILe989MXipoE9332s=
-k8s.io/client-go v0.22.8/go.mod h1:dOHOy82WOBz0siYHpVyY7FqTIq+iXFXW3+THFk6qErU=
-k8s.io/component-base v0.22.8/go.mod h1:rt0sHx3GT3i8e72rPKQSrTIV3THWyEl2IDYcnEbxraI=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
+k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
+k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
+k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
+k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
+k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
-k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kubelet v0.22.8 h1:Ce1yoWCd03Gp5Dqe4BUy1FlQntyIUfCUcL8E6mKIgi4=
-k8s.io/kubelet v0.22.8/go.mod h1:IHfh4AnROFQ6fMnCeoTQIrnF0H148PlfllXeupW698c=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
+k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kubelet v0.27.5 h1:uysO9NozKUi5zAde+hMXfCU1dWNjL/UBhRGVZk8uUJQ=
+k8s.io/kubelet v0.27.5/go.mod h1:xwIXdhJReWW2GuFQpAlj1qbaxD1O7JpGueItvc47tXg=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

hack/build-go.sh

@@ -57,22 +57,28 @@ LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version=${VER
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.gitTreeState=${GIT_TREE_STATE} \
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.releaseStatus=${RELEASE_STATUS} \
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date=${DATE}"
-export CGO_ENABLED=0
+export CGO_ENABLED=${CGO_ENABLED:-0}
 
 # build with go modules
 export GO111MODULE=on
-BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+
 if [ -n "$MODMODE" ]; then
-	BUILD_ARGS+=(-mod "$MODMODE")
+	BUILD_ARGS=(-mod "$MODMODE")
 fi
 
 echo "Building multus"
-go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
+go build -o ${DEST_DIR}/multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
 echo "Building multus-daemon"
-go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/multus-daemon
+go build -o "${DEST_DIR}"/multus-daemon ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-daemon
 echo "Building multus-shim"
-go build -o "${DEST_DIR}"/multus-shim -ldflags "${LDFLAGS}" ./cmd/multus-shim
+go build -o "${DEST_DIR}"/multus-shim ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-shim
 echo "Building install_multus"
-go build -o "${DEST_DIR}"/install_multus -ldflags "${LDFLAGS}" ./cmd/install_multus
+go build -o "${DEST_DIR}"/install_multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/install_multus
 echo "Building thin_entrypoint"
-go build -o "${DEST_DIR}"/thin_entrypoint -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
+go build -o "${DEST_DIR}"/thin_entrypoint ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
+echo "Building kubeconfig_generator"
+go build -o "${DEST_DIR}"/kubeconfig_generator ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/kubeconfig_generator
+echo "Building cert-approver"
+go build -o "${DEST_DIR}"/cert-approver ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/cert-approver
+echo "Building passthru CNI"
+go build -o "${DEST_DIR}"/passthru ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/passthru-cni

hack/test-go.sh

@@ -19,5 +19,5 @@ if [ "$GO111MODULE" == "off" ]; then
 	bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
 else
 	# test with go modules
-	bash -c "umask 0; go test -v -covermode=count -coverprofile=coverage.out ./..."
+	bash -c "umask 0; go test -v -race -covermode=atomic -coverprofile=coverage.out ./..."
 fi

images/Dockerfile

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.23 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -8,7 +8,7 @@ ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM gcr.io/distroless/base-debian11:latest
+FROM gcr.io/distroless/base-debian12:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
@@ -16,4 +16,7 @@ WORKDIR /
 
 COPY --from=build /usr/src/multus-cni/bin/install_multus /
 COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+
 ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.debug

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.23 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -8,7 +8,7 @@ ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM gcr.io/distroless/base-debian11:debug
+FROM gcr.io/distroless/base-debian12:debug
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
@@ -16,4 +16,7 @@ WORKDIR /
 
 COPY --from=build /usr/src/multus-cni/bin/install_multus /
 COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+
 ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.thick

@@ -1,16 +1,18 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.23 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
+ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 FROM debian:stable-slim
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+LABEL org.opencontainers.image.source=https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
 WORKDIR /
 
 ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

pkg/checkpoint/checkpoint_test.go

@@ -15,6 +15,8 @@
 
 package checkpoint
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"

pkg/cmdutils/cmdutils_suite_test.go

@@ -15,6 +15,8 @@
 // Package cmdutils is the package that contains utilities for multus command
 package cmdutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/cmdutils/utils_test.go

@@ -15,6 +15,8 @@
 // Package cmdutils is the package that contains utilities for multus command
 package cmdutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"

pkg/k8sclient/k8sclient.go

@@ -18,29 +18,29 @@ package k8sclient
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
+	"syscall"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
+	listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/klog"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
-	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/kubeletclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
@@ -61,9 +61,13 @@ type NoK8sNetworkError struct {
 // ClientInfo contains information given from k8s client
 type ClientInfo struct {
 	Client           kubernetes.Interface
-	NetClient        netclient.K8sCniCncfIoV1Interface
+	NetClient        netclient.Interface
 	EventBroadcaster record.EventBroadcaster
 	EventRecorder    record.EventRecorder
+
+	// multus-thick uses these informer
+	PodInformer    cache.SharedIndexInformer
+	NetDefInformer cache.SharedIndexInformer
 }
 
 // AddPod adds pod into kubernetes
@@ -73,9 +77,27 @@ func (c *ClientInfo) AddPod(pod *v1.Pod) (*v1.Pod, error) {
 
 // GetPod gets pod from kubernetes
 func (c *ClientInfo) GetPod(namespace, name string) (*v1.Pod, error) {
+	if c.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
+		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
+	}
 	return c.Client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
+// GetPodContext gets pod from kubernetes with context
+func (c *ClientInfo) GetPodContext(ctx context.Context, namespace, name string) (*v1.Pod, error) {
+	if c.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
+		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
+	}
+	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+}
+
+// GetPodAPILiveQuery does a live API query for the pod, instead of using informers, for cases when a failure occurred, as to prevent a cache miss.
+func (c *ClientInfo) GetPodAPILiveQuery(ctx context.Context, namespace, name string) (*v1.Pod, error) {
+	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+}
+
 // DeletePod deletes a pod from kubernetes
 func (c *ClientInfo) DeletePod(namespace, name string) error {
 	return c.Client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
@@ -83,7 +105,16 @@ func (c *ClientInfo) DeletePod(namespace, name string) error {
 
 // AddNetAttachDef adds net-attach-def into kubernetes
 func (c *ClientInfo) AddNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) (*nettypes.NetworkAttachmentDefinition, error) {
-	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
+	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
+}
+
+// GetNetAttachDef get net-attach-def from kubernetes
+func (c *ClientInfo) GetNetAttachDef(namespace, name string) (*nettypes.NetworkAttachmentDefinition, error) {
+	if c.NetDefInformer != nil {
+		logging.Debugf("GetNetAttachDef for [%s/%s] will use informer cache", namespace, name)
+		return netlister.NewNetworkAttachmentDefinitionLister(c.NetDefInformer.GetIndexer()).NetworkAttachmentDefinitions(namespace).Get(name)
+	}
+	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
 // Eventf puts event into kubernetes events
@@ -113,7 +144,7 @@ func SetPodNetworkStatusAnnotation(client *ClientInfo, podName string, podNamesp
 	if err != nil {
 		return logging.Errorf("SetNetworkStatus: %v", err)
 	}
-	if client == nil || client.Client == nil {
+	if client == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
 			return logging.Errorf("SetNetworkStatus: must have either Kubernetes config or delegates")
@@ -168,16 +199,22 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	// Check and see if each item matches the specification for valid attachment name.
 	// "Valid attachment names must be comprised of units of the DNS-1123 label format"
 	// [a-z0-9]([-a-z0-9]*[a-z0-9])?
-	// And we allow at (@), and forward slash (/) (units separated by commas)
 	// It must start and end alphanumerically.
-	allItems := []string{netNsName, networkName, netIfName}
+	allItems := []string{netNsName, networkName}
+	expr := regexp.MustCompile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
 	for i := range allItems {
-		matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i])
+		matched := expr.MatchString(allItems[i])
 		if !matched && len([]rune(allItems[i])) > 0 {
 			return "", "", "", logging.Errorf(fmt.Sprintf("parsePodNetworkObjectName: Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
 		}
 	}
 
+	if len(netIfName) > 0 {
+		if len(netIfName) > (syscall.IFNAMSIZ-1) || strings.ContainsAny(netIfName, " \t\n\v\f\r/") {
+			return "", "", "", logging.Errorf(fmt.Sprintf("parsePodNetworkObjectName: Failed to parse interface name: must be less than 15 chars and not contain '/' or spaces. interface name '%s'", netIfName))
+		}
+	}
+
 	logging.Debugf("parsePodNetworkObjectName: parsed: %s, %s, %s", netNsName, networkName, netIfName)
 	return netNsName, networkName, netIfName, nil
 }
@@ -254,7 +291,8 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 
 	logging.Debugf("getKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
-	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(context.TODO(), net.Name, metav1.GetOptions{})
+
+	customResource, err := client.GetNetAttachDef(net.Namespace, net.Name)
 	if err != nil {
 		errMsg := fmt.Sprintf("cannot find a network-attachment-definition (%s) in namespace (%s): %v", net.Name, net.Namespace, err)
 		if client != nil {
@@ -266,7 +304,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 	// Get resourceName annotation from NetworkAttachmentDefinition
 	deviceID := ""
 	resourceName, ok := customResource.GetAnnotations()[resourceNameAnnot]
-	if ok && pod.Name != "" && pod.Namespace != "" {
+	if ok && pod != nil && pod.Name != "" && pod.Namespace != "" {
 		// ResourceName annotation is found; try to get device info from resourceMap
 		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
 
@@ -292,11 +330,6 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		}
 	}
 
-	// acquire lock to access file
-	if types.ChrootMutex != nil {
-		types.ChrootMutex.Lock()
-		defer types.ChrootMutex.Unlock()
-	}
 	configBytes, err := netutils.GetCNIConfig(customResource, confdir)
 	if err != nil {
 		return nil, resourceMap, err
@@ -393,82 +426,6 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 	return 0, clientInfo, err
 }
 
-// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
-// the k8s API.
-func InClusterK8sClient() (*ClientInfo, error) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	logging.Debugf("InClusterK8sClient: in cluster config: %+v", config)
-	return NewClientInfo(config)
-}
-
-// GetK8sClient gets client info from kubeconfig
-func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
-	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
-	// If we get a valid kubeClient (eg from testcases) just return that
-	// one.
-	if kubeClient != nil {
-		return kubeClient, nil
-	}
-
-	var err error
-	var config *rest.Config
-
-	// Otherwise try to create a kubeClient from a given kubeConfig
-	if kubeconfig != "" {
-		// uses the current context in kubeconfig
-		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
-		}
-	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
-		// Try in-cluster config where multus might be running in a kubernetes pod
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
-		}
-	} else {
-		// No kubernetes config; assume we shouldn't talk to Kube at all
-		return nil, nil
-	}
-
-	// Specify that we use gRPC
-	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
-	config.ContentType = "application/vnd.kubernetes.protobuf"
-	// Set the config timeout to one minute.
-	config.Timeout = time.Minute
-
-	return NewClientInfo(config)
-}
-
-// NewClientInfo returns a `ClientInfo` from a configuration created from an
-// existing kubeconfig file.
-func NewClientInfo(config *rest.Config) (*ClientInfo, error) {
-	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	netclient, err := netclient.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(klog.Infof)
-	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
-	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
-	return &ClientInfo{
-		Client:           client,
-		NetClient:        netclient,
-		EventBroadcaster: broadcaster,
-		EventRecorder:    recorder,
-	}, nil
-}
-
 // GetPodNetwork gets net-attach-def annotation from pod
 func GetPodNetwork(pod *v1.Pod) ([]*types.NetworkSelectionElement, error) {
 	logging.Debugf("GetPodNetwork: %v", pod)
@@ -548,12 +505,6 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 
 		// option2) search CNI json config file, which has <netname> as CNI name, from confDir
 
-		// acquire lock to access file
-		if types.ChrootMutex != nil {
-			types.ChrootMutex.Lock()
-			defer types.ChrootMutex.Unlock()
-		}
-
 		configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
 		if err == nil {
 			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
@@ -563,12 +514,6 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 			return delegate, resourceMap, nil
 		}
 	} else {
-		// acquire lock to access file
-		if types.ChrootMutex != nil {
-			types.ChrootMutex.Lock()
-			defer types.ChrootMutex.Unlock()
-		}
-
 		fInfo, err := os.Stat(netname)
 		if err != nil {
 			return nil, resourceMap, err
@@ -595,31 +540,119 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 		} else {
 			// option4) if file path (absolute), then load it directly
 			if strings.HasSuffix(netname, ".conflist") {
-				confList, err := libcni.ConfListFromFile(netname)
+				confList, err := LoadChainedPluginsFromFile(netname)
 				if err != nil {
 					return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
 				}
-				configBytes = confList.Bytes
-			} else {
-				conf, err := libcni.ConfFromFile(netname)
+
+				delegate, err := types.LoadDelegateNetConfFromConfList(confList, nil, "", "")
 				if err != nil {
-					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
+					return nil, resourceMap, err
 				}
-				if conf.Network.Type == "" {
-					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
-				}
-				configBytes = conf.Bytes
+				return delegate, resourceMap, nil
+
 			}
-			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
+
+			// Or it's not a conflist...
+			// after libcni v1.2.3 there's no support support this old-school method with non-conflists.
+			// this method doesn't check if there's a 0 length plugins field, that is.
+			conf, err := libcni.ConfFromFile(netname)
+			if err != nil {
+				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
+			}
+			if conf.Network.Type == "" {
+				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is supposed to be a .conflist?", netname)
+			}
+
+			delegate, err := types.LoadDelegateNetConf(conf.Bytes, nil, "", "")
 			if err != nil {
 				return nil, resourceMap, err
 			}
 			return delegate, resourceMap, nil
 		}
+
 	}
 	return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
 }
 
+func loadSubdirectoryChain(bytes []byte, cniconfdir string) (*libcni.NetworkConfigList, error) {
+	// Load the network configuration from the byte array
+	conf, err := libcni.NetworkConfFromBytes(bytes)
+	if err != nil {
+		return nil, fmt.Errorf("error loading network config from bytes: %v", err)
+	}
+
+	// Check if plugins need to be loaded from files
+	if !conf.LoadOnlyInlinedPlugins && cniconfdir != "" {
+		// Let's validate that conf.Name
+		// From the CNI spec:
+		// > Must start with an alphanumeric character, optionally followed by any combination of one or more alphanumeric characters,
+		// > underscore, dot (.) or hyphen (-). Must not contain characters disallowed in file paths.
+		if !regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`).MatchString(conf.Name) {
+			return nil, fmt.Errorf("invalid network config name: %s", conf.Name)
+		}
+
+		plugins, err := libcni.NetworkPluginConfsFromFiles(cniconfdir, conf.Name)
+		if err != nil {
+			return nil, fmt.Errorf("error loading plugin configs: %v", err)
+		}
+		conf.Plugins = append(conf.Plugins, plugins...)
+	}
+
+	if len(conf.Plugins) == 0 {
+		return nil, fmt.Errorf("no plugin configs found")
+	}
+
+	return conf, nil
+}
+
+// LoadChainedDelegatesFromBytes loads a CNI configuration byte array and returns a DelegateNetConf with the chain added.
+func LoadChainedDelegatesFromBytes(bytes []byte, cniconfdir string) *types.DelegateNetConf {
+	conf, err := loadSubdirectoryChain(bytes, cniconfdir)
+	if err != nil {
+		logging.Errorf("LoadChainedDelegatesFromBytes: %v", err)
+		return nil
+	}
+
+	// Create and return a DelegateNetConf from the configuration list
+	delegate, err := types.LoadDelegateNetConfFromConfList(conf, nil, "", "")
+	if err != nil {
+		logging.Errorf("LoadChainedDelegatesFromBytes: error loading delegate network config: %v", err)
+		return nil
+	}
+
+	return delegate
+}
+
+// LoadChainedPluginsFromFile loads a CNI configuration file and returns the NetworkConfigList
+func LoadChainedPluginsFromFile(filename string) (*libcni.NetworkConfigList, error) {
+	cleanPath := filepath.Clean(filename)
+
+	// stat the file to make sure it's a normal file.
+	info, err := os.Stat(cleanPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.Mode().IsRegular() {
+		return nil, errors.New("CNI configuration path is not a regular file")
+	}
+
+	bytes, err := os.ReadFile(cleanPath)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", filename, err)
+	}
+	logging.Debugf("LoadChainedPluginsFromFile: %s", filename)
+
+	conf, err := loadSubdirectoryChain(bytes, filepath.Dir(filename))
+	if err != nil {
+		return nil, err
+	}
+	logging.Debugf("Loaded SubdirectoryChain: %+v", conf)
+
+	return conf, nil
+}
+
 // GetDefaultNetworks parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
 func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo, resourceMap map[string]*types.ResourceInfo) (map[string]*types.ResourceInfo, error) {
 	logging.Debugf("GetDefaultNetworks: %v, %v, %v, %v", pod, conf, kubeClient, resourceMap)
@@ -646,7 +679,7 @@ func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo
 	delegates = append(delegates, delegate)
 
 	// Pod in kube-system namespace does not have default network for now.
-	if !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
+	if pod != nil && !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
 		for _, netname := range conf.DefaultNetworks {
 			delegate, resourceMap, err := getNetDelegate(kubeClient, pod, netname, conf.ConfDir, conf.MultusNamespace, resourceMap)
 			if err != nil {

pkg/k8sclient/k8sclient_test.go

@@ -15,6 +15,8 @@
 
 package k8sclient
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
@@ -46,7 +48,7 @@ func TestK8sClient(t *testing.T) {
 func NewFakeClientInfo() *ClientInfo {
 	return &ClientInfo{
 		Client:    fake.NewSimpleClientset(),
-		NetClient: netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient: netfake.NewSimpleClientset(),
 	}
 }
 
@@ -1000,36 +1002,30 @@ users:
 	})
 
 	Context("parsePodNetworkObjectName", func() {
-		It("fails to get podnetwork given bad annotation values", func() {
-			fakePod := testutils.NewFakePod(fakePodName, "net1", "")
-
-			clientInfo := NewFakeClientInfo()
-			_, err := clientInfo.AddPod(fakePod)
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = clientInfo.AddNetAttachDef(
-				testutils.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", "{\"type\": \"mynet\"}"))
-			Expect(err).NotTo(HaveOccurred())
-
-			k8sArgs, err := GetK8sArgs(args)
-			Expect(err).NotTo(HaveOccurred())
-			pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
-			// invalid case 1 - can't have more than 2 items separated by "/"
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP/root@thirdIP"
+		DescribeTable("fails to get podnetwork given bad annotation values", func(networkAnnot string) {
+			pod := testutils.NewFakePod(fakePodName, "net1", "")
+			pod.Annotations[networkAttachmentAnnot] = networkAnnot
 			_, err = GetPodNetwork(pod)
 			Expect(err).To(HaveOccurred())
+		},
+			Entry("can't have more than 2 items separated by \"/\"", "root@someIP/root@someOtherIP/root@thirdIP"),
+			Entry("can't have more than 2 items separated by \"@\"", "root@someIP/root@someOtherIP@garbagevalue"),
+			Entry("not matching comma-delimited format", "root@someIP/root@someOtherIP"),
+			Entry("invalid network interface name space in netdev name", "default/net1@myIfc Name"),
+			Entry("invalid network interface name too long", "default/net1@very_long_interface_name"),
+		)
 
-			// invalid case 2 - can't have more than 2 items separated by "@"
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP@garbagevalue"
+		DescribeTable("gets pod network successfully from annotation values", func(networkAnnot string) {
+			pod := testutils.NewFakePod(fakePodName, "net1", "")
+			pod.Annotations[networkAttachmentAnnot] = networkAnnot
 			_, err = GetPodNetwork(pod)
-			Expect(err).To(HaveOccurred())
-
-			// invalid case 3 - not matching comma-delimited format
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP"
-			_, err = GetPodNetwork(pod)
-			Expect(err).To(HaveOccurred())
-		})
+			Expect(err).ToNot(HaveOccurred())
+		},
+			Entry("network without namespace", "net1"),
+			Entry("network with namespace", "default/net1"),
+			Entry("network with interface name", "net1@my_interface"),
+			Entry("network with interface name and namespace", "default/net1@my_interface"),
+		)
 	})
 
 	Context("setPodNetworkAnnotation", func() {
@@ -1203,11 +1199,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1258,11 +1257,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1316,11 +1318,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1398,11 +1403,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1454,11 +1462,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1509,11 +1520,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 

pkg/k8sclient/kubeconfig.go

@@ -0,0 +1,247 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8sclient
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/transport"
+	"k8s.io/client-go/util/certificate"
+	"k8s.io/klog"
+
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+)
+
+const (
+	certNamePrefix       = "multus-client"
+	certCommonNamePrefix = "system:multus"
+	certOrganization     = "system:multus"
+)
+
+var (
+	certUsages = []certificatesv1.KeyUsage{certificatesv1.UsageDigitalSignature, certificatesv1.UsageClientAuth}
+)
+
+// getPerNodeKubeconfig creates new kubeConfig, based on bootstrap, with new certDir
+func getPerNodeKubeconfig(bootstrap *rest.Config, certDir string) *rest.Config {
+	return &rest.Config{
+		Host:    bootstrap.Host,
+		APIPath: bootstrap.APIPath,
+		ContentConfig: rest.ContentConfig{
+			AcceptContentTypes: "application/vnd.kubernetes.protobuf,application/json",
+			ContentType:        "application/vnd.kubernetes.protobuf",
+		},
+		TLSClientConfig: rest.TLSClientConfig{
+			KeyFile:  path.Join(certDir, certNamePrefix+"-current.pem"),
+			CertFile: path.Join(certDir, certNamePrefix+"-current.pem"),
+			CAData:   bootstrap.TLSClientConfig.CAData,
+		},
+		// Allow multus (especially in server mode) to make more concurrent requests
+		// to reduce client-side throttling
+		QPS:   50,
+		Burst: 50,
+		// Set the config timeout to one minute.
+		Timeout: time.Minute,
+	}
+}
+
+// PerNodeK8sClient creates/reload new multus kubeconfig per-node.
+func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration time.Duration, certDir string) (*ClientInfo, error) {
+	bootstrapKubeconfig, err := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+	if err != nil {
+		return nil, logging.Errorf("failed to load bootstrap kubeconfig %s: %v", bootstrapKubeconfigFile, err)
+	}
+	config := getPerNodeKubeconfig(bootstrapKubeconfig, certDir)
+
+	// If we have a valid certificate, user that to fetch CSRs.
+	// Otherwise, use the bootstrap credentials from bootstrapKubeconfig
+	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
+	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
+		cfg := bootstrapKubeconfig
+
+		// validate the kubeconfig
+		tempClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
+		} else {
+			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+			// tls unknown authority error is unrecoverable error with retry
+			if err != nil {
+				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
+					// reload and use bootstrapKubeconfig again
+					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					cfg = newBootstrapKubeconfig
+				} else {
+					logging.Errorf("failed to list pods with new certs: %v", err)
+				}
+			}
+
+			if current != nil {
+				cfg = config
+			}
+		}
+		return kubernetes.NewForConfig(cfg)
+	}
+
+	certificateStore, err := certificate.NewFileStore(certNamePrefix, certDir, certDir, "", "")
+	if err != nil {
+		return nil, logging.Errorf("failed to initialize the certificate store: %v", err)
+	}
+
+	certManager, err := certificate.NewManager(&certificate.Config{
+		ClientsetFn: newClientsetFn,
+		Template: &x509.CertificateRequest{
+			Subject: pkix.Name{
+				CommonName:   fmt.Sprintf("%s:%s", certCommonNamePrefix, nodeName),
+				Organization: []string{certOrganization},
+			},
+		},
+		RequestedCertificateLifetime: &certDuration,
+		SignerName:                   certificatesv1.KubeAPIServerClientSignerName,
+		Usages:                       certUsages,
+		CertificateStore:             certificateStore,
+	})
+	if err != nil {
+		return nil, logging.Errorf("failed to initialize the certificate manager: %v", err)
+	}
+	if certDuration < time.Hour {
+		// the default value for CertCallbackRefreshDuration (5min) is too long for short-lived certs,
+		// set it to a more sensible value
+		transport.CertCallbackRefreshDuration = time.Second * 10
+	}
+	certManager.Start()
+
+	logging.Verbosef("Waiting for certificate")
+	var storeErr error
+	err = wait.PollWithContext(context.TODO(), time.Second, 2*time.Minute, func(_ context.Context) (bool, error) {
+		var currentCert *tls.Certificate
+		currentCert, storeErr = certificateStore.Current()
+		return currentCert != nil && storeErr == nil, nil
+	})
+	if err != nil {
+		return nil, logging.Errorf("certificate was not signed, last cert store err: %v err: %v", storeErr, err)
+	}
+	logging.Verbosef("Certificate found!")
+
+	return newClientInfo(config)
+}
+
+// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
+// the k8s API.
+func InClusterK8sClient() (*ClientInfo, error) {
+	clientInfo, err := GetK8sClient("", nil)
+	if err != nil {
+		return nil, err
+	}
+	if clientInfo == nil {
+		return nil, fmt.Errorf("failed to create in-cluster kube client")
+	}
+	return clientInfo, err
+}
+
+// SetK8sClientInformers adds informer structure to ClientInfo to utilize in thick daemon
+func (c *ClientInfo) SetK8sClientInformers(podInformer, netDefInformer cache.SharedIndexInformer) {
+	c.PodInformer = podInformer
+	c.NetDefInformer = netDefInformer
+}
+
+// GetK8sClient gets client info from kubeconfig
+func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
+	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
+	// If we get a valid kubeClient (eg from testcases) just return that
+	// one.
+	if kubeClient != nil {
+		return kubeClient, nil
+	}
+
+	var err error
+	var config *rest.Config
+
+	// Otherwise try to create a kubeClient from a given kubeConfig
+	if kubeconfig != "" {
+		// uses the current context in kubeconfig
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
+		}
+	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
+		// Try in-cluster config where multus might be running in a kubernetes pod
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
+		}
+	} else {
+		// No kubernetes config; assume we shouldn't talk to Kube at all
+		return nil, nil
+	}
+
+	// Specify that we use gRPC
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+	// Set the config timeout to one minute.
+	config.Timeout = time.Minute
+	// Allow multus (especially in server mode) to make more concurrent requests
+	// to reduce client-side throttling
+	config.QPS = 50
+	config.Burst = 50
+
+	return newClientInfo(config)
+}
+
+// newClientInfo returns a `ClientInfo` from a configuration created from an
+// existing kubeconfig file.
+func newClientInfo(config *rest.Config) (*ClientInfo, error) {
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	netclient, err := netclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
+	return &ClientInfo{
+		Client:           client,
+		NetClient:        netclient,
+		EventBroadcaster: broadcaster,
+		EventRecorder:    recorder,
+	}, nil
+}

pkg/kubeletclient/kubeletclient.go

@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"golang.org/x/net/context"
@@ -137,19 +138,45 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou
 	for _, pr := range rc.resources {
 		if pr.Name == name && pr.Namespace == ns {
 			for _, cnt := range pr.Containers {
-				for _, dev := range cnt.Devices {
-					if rInfo, ok := resourceMap[dev.ResourceName]; ok {
-						rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...)
-					} else {
-						resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds}
-					}
-				}
+				rc.getDevicePluginResources(cnt.Devices, resourceMap)
+				rc.getDRAResources(cnt.DynamicResources, resourceMap)
 			}
 		}
 	}
 	return resourceMap, nil
 }
 
+func (rc *kubeletClient) getDevicePluginResources(devices []*podresourcesapi.ContainerDevices, resourceMap map[string]*types.ResourceInfo) {
+	for _, dev := range devices {
+		if rInfo, ok := resourceMap[dev.ResourceName]; ok {
+			rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...)
+		} else {
+			resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds}
+		}
+	}
+}
+
+func (rc *kubeletClient) getDRAResources(dynamicResources []*podresourcesapi.DynamicResource, resourceMap map[string]*types.ResourceInfo) {
+	for _, dynamicResource := range dynamicResources {
+		var deviceIDs []string
+		for _, claimResource := range dynamicResource.ClaimResources {
+			for _, cdiDevice := range claimResource.CDIDevices {
+				res := strings.Split(cdiDevice.Name, "=")
+				if len(res) == 2 {
+					deviceIDs = append(deviceIDs, res[1])
+				} else {
+					logging.Errorf("GetPodResourceMap: Invalid CDI format")
+				}
+			}
+		}
+		if rInfo, ok := resourceMap[dynamicResource.ClassName]; ok {
+			rInfo.DeviceIDs = append(rInfo.DeviceIDs, deviceIDs...)
+		} else {
+			resourceMap[dynamicResource.ClassName] = &types.ResourceInfo{DeviceIDs: deviceIDs}
+		}
+	}
+}
+
 func hasKubeletAPIEndpoint(url *url.URL) bool {
 	// Check for kubelet resource API socket file
 	if _, err := os.Stat(url.Path); err != nil {

pkg/kubeletclient/kubeletclient_test.go

@@ -15,6 +15,8 @@
 
 package kubeletclient
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -52,11 +54,12 @@ func (m *fakeResourceServer) GetAllocatableResources(_ context.Context, _ *podre
 	return &podresourcesapi.AllocatableResourcesResponse{}, nil
 }
 
-func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
-	podName := "pod-name"
-	podNamespace := "pod-namespace"
-	containerName := "container-name"
+// TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
+func (m *fakeResourceServer) Get(_ context.Context, _ *podresourcesapi.GetPodResourcesRequest) (*podresourcesapi.GetPodResourcesResponse, error) {
+	return &podresourcesapi.GetPodResourcesResponse{}, nil
+}
 
+func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
 	devs := []*podresourcesapi.ContainerDevices{
 		{
 			ResourceName: "resource",
@@ -64,18 +67,49 @@ func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodR
 		},
 	}
 
+	cdiDevices := []*podresourcesapi.CDIDevice{
+		{
+			Name: "cdi-kind=cdi-resource",
+		},
+	}
+
+	claimsResource := []*podresourcesapi.ClaimResource{
+		{
+			CDIDevices: cdiDevices,
+		},
+	}
+
+	dynamicResources := []*podresourcesapi.DynamicResource{
+		{
+			ClassName:      "resource-class",
+			ClaimName:      "resource-claim",
+			ClaimNamespace: "dynamic-resource-pod-namespace",
+			ClaimResources: claimsResource,
+		},
+	}
+
 	resp := &podresourcesapi.ListPodResourcesResponse{
 		PodResources: []*podresourcesapi.PodResources{
 			{
-				Name:      podName,
-				Namespace: podNamespace,
+				Name:      "pod-name",
+				Namespace: "pod-namespace",
 				Containers: []*podresourcesapi.ContainerResources{
 					{
-						Name:    containerName,
+						Name:    "container-name",
 						Devices: devs,
 					},
 				},
 			},
+			{
+				Name:      "dynamic-resource-pod-name",
+				Namespace: "dynamic-resource-pod-namespace",
+				Containers: []*podresourcesapi.ContainerResources{
+					{
+						Name:             "dynamic-resource-container-name",
+						DynamicResources: dynamicResources,
+					},
+				},
+			},
 		},
 	}
 	return resp, nil
@@ -181,7 +215,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 		})
 	})
 	Context("GetPodResourceMap() with valid pod name and namespace", func() {
-		It("should return no error", func() {
+		It("should return no error with device plugin resource", func() {
 			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
 			fakePod := &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
@@ -209,6 +243,34 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 			Expect(resourceMap).To(Equal(outputRMap))
 		})
 
+		It("should return no error with dynamic resource", func() {
+			podUID := k8sTypes.UID("9f94e27b-4233-43d6-bd10-f73b4de6f456")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "dynamic-resource-pod-name",
+					Namespace: "dynamic-resource-pod-namespace",
+					UID:       podUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "dynamic-resource-container-name",
+						},
+					},
+				},
+			}
+			client, err := getKubeletClient(testKubeletSocket)
+			Expect(err).NotTo(HaveOccurred())
+
+			outputRMap := map[string]*mtypes.ResourceInfo{
+				"resource-class": {DeviceIDs: []string{"cdi-resource"}},
+			}
+			resourceMap, err := client.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(resourceMap).ShouldNot(BeNil())
+			Expect(resourceMap).To(Equal(outputRMap))
+		})
+
 		It("should return an error with garbage socket value", func() {
 			u, err := url.Parse("/badfilepath!?//")
 			Expect(err).NotTo(HaveOccurred())

pkg/logging/logging.go

@@ -56,25 +56,35 @@ type LogOptions struct {
 
 // SetLogOptions set the LoggingOptions of NetConf
 func SetLogOptions(options *LogOptions) {
+	// logger is used only if filname is supplied
+	if logger == nil || logger.Filename == "" {
+		return
+	}
+
 	// give some default value
-	logger.MaxSize = 100
-	logger.MaxAge = 5
-	logger.MaxBackups = 5
-	logger.Compress = true
+	updatedLogger := lumberjack.Logger{
+		Filename:   logger.Filename,
+		MaxAge:     5,
+		MaxBackups: 5,
+		Compress:   true,
+		MaxSize:    100,
+		LocalTime:  logger.LocalTime,
+	}
 	if options != nil {
 		if options.MaxAge != nil {
-			logger.MaxAge = *options.MaxAge
+			updatedLogger.MaxAge = *options.MaxAge
 		}
 		if options.MaxSize != nil {
-			logger.MaxSize = *options.MaxSize
+			updatedLogger.MaxSize = *options.MaxSize
 		}
 		if options.MaxBackups != nil {
-			logger.MaxBackups = *options.MaxBackups
+			updatedLogger.MaxBackups = *options.MaxBackups
 		}
 		if options.Compress != nil {
-			logger.Compress = *options.Compress
+			updatedLogger.Compress = *options.Compress
 		}
 	}
+	logger = &updatedLogger
 	loggingW = logger
 }
 
@@ -171,18 +181,32 @@ func SetLogStderr(enable bool) {
 
 // SetLogFile sets logging file
 func SetLogFile(filename string) {
+	// logger is used only if filname is supplied
 	if filename == "" {
 		return
 	}
 
-	logger.Filename = filename
-	loggingW = logger
+	updatedLogger := lumberjack.Logger{
+		Filename:   filename,
+		MaxAge:     5,
+		MaxBackups: 5,
+		Compress:   true,
+		MaxSize:    100,
+	}
 
+	if logger != nil {
+		updatedLogger.MaxAge = logger.MaxAge
+		updatedLogger.MaxBackups = logger.MaxBackups
+		updatedLogger.Compress = logger.Compress
+		updatedLogger.MaxSize = logger.MaxSize
+	}
+	logger = &updatedLogger
+	loggingW = logger
 }
 
 func init() {
 	loggingStderr = true
 	loggingW = nil
 	loggingLevel = PanicLevel
-	logger = &lumberjack.Logger{}
+	logger = nil
 }

pkg/logging/logging_test.go

@@ -15,6 +15,8 @@
 
 package logging
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
@@ -106,13 +108,13 @@ var _ = Describe("logging operations", func() {
 		Verbosef("foobar")
 		Expect(Errorf("foobar")).NotTo(BeNil())
 		Panicf("foobar")
-		logger.Filename = ""
+		logger = nil
 		loggingW = nil
 		err = os.RemoveAll(tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		// Revert the log variable to init
 		loggingW = nil
-		logger = &lumberjack.Logger{}
+		logger = nil
 	})
 
 	// Tests public getter

pkg/multus/multus.go

@@ -46,21 +46,18 @@ import (
 )
 
 const (
-	shortPollDuration = 250 * time.Millisecond
-	shortPollTimeout  = 2500 * time.Millisecond
+	shortPollDuration    = 250 * time.Millisecond
+	informerPollDuration = 50 * time.Millisecond
+	shortPollTimeout     = 2500 * time.Millisecond
 )
 
 var (
-	version       = "master@git"
-	commit        = "unknown commit"
-	date          = "unknown date"
-	gitTreeState  = ""
-	releaseStatus = ""
-)
-
-var (
-	pollDuration = 1000 * time.Millisecond
-	pollTimeout  = 45 * time.Second
+	version        = "master@git"
+	commit         = "unknown commit"
+	date           = "unknown date"
+	gitTreeState   = ""
+	releaseStatus  = ""
+	errPodNotFound = fmt.Errorf("pod not found during Multus GetPod")
 )
 
 // PrintVersionString ...
@@ -135,15 +132,49 @@ func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetCo
 	return err
 }
 
-func deleteDelegates(containerID, dataDir string) error {
-	logging.Debugf("deleteDelegates: %s, %s", containerID, dataDir)
-
-	path := filepath.Join(dataDir, containerID)
-	if err := os.Remove(path); err != nil {
-		return logging.Errorf("deleteDelegates: error in deleting the delegates : %v", err)
+func getValidAttachmentFromCache(b []byte) (string, string, error) {
+	type simpleCacheV1 struct {
+		Kind        string `json:"kind"`
+		ContainerID string `json:"containerId"`
+		IfName      string `json:"ifName"`
 	}
 
-	return nil
+	cache := &simpleCacheV1{}
+	if err := json.Unmarshal(b, cache); err != nil {
+		return "", "", fmt.Errorf("getValidAttachmentFromCache: invalid json: %v", err)
+	}
+
+	if cache.ContainerID == "" || cache.IfName == "" {
+		return "", "", fmt.Errorf("invalid cache: containerID:%q, ifName:%q", cache.ContainerID, cache.IfName)
+	}
+
+	return cache.ContainerID, cache.IfName, nil
+}
+
+func gatherValidAttachmentsFromCache(cniDir string) ([]cnitypes.GCAttachment, error) {
+	cacheDir := filepath.Join(cniDir, "results")
+	dirEntries, err := os.ReadDir(cacheDir)
+	if err != nil {
+		return nil, err
+	}
+
+	allAttachments := []cnitypes.GCAttachment{}
+	for _, dirEnt := range dirEntries {
+		path := filepath.Join(cacheDir, dirEnt.Name())
+		delegatesBytes, err := os.ReadFile(path)
+		// if delegates cannot read that, skipped for now (because cannot recover).
+		if err != nil {
+			logging.Errorf("gatherSavedDelegates: cannot read %q, skipped to add", path)
+			continue
+		}
+		containerID, ifName, err := getValidAttachmentFromCache(delegatesBytes)
+		if err != nil {
+			logging.Errorf("gatherSavedDelegates: cannot read cache, skipped to add: %v", err)
+			continue
+		}
+		allAttachments = append(allAttachments, cnitypes.GCAttachment{ContainerID: containerID, IfName: ifName})
+	}
+	return allAttachments, nil
 }
 
 func validateIfName(nsname string, ifname string) error {
@@ -227,16 +258,25 @@ func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.Net
 	return err
 }
 
-func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
+func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, cniConfList *libcni.NetworkConfigList, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
 	logging.Debugf("conflistAdd: %v, %s", rt, string(rawnetconflist))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
 	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
 	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
-	confList, err := libcni.ConfListFromBytes(rawnetconflist)
-	if err != nil {
-		return nil, logging.Errorf("conflistAdd: error converting the raw bytes into a conflist: %v", err)
+	var confList *libcni.NetworkConfigList
+	var err error
+
+	// This may wind up being set during parsing the default network config.
+	// In this case -- we'll use it as passed. Otherwise, we'll recalculate it.
+	if len(cniConfList.Plugins) > 0 {
+		confList = cniConfList
+	} else {
+		confList, err = libcni.NetworkConfFromBytes(rawnetconflist)
+		if err != nil {
+			return nil, logging.Errorf("conflistAdd: error converting the raw bytes into a conflist: %v", err)
+		}
 	}
 
 	result, err := cniNet.AddNetworkList(context.Background(), confList, rt)
@@ -330,7 +370,8 @@ func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 	var result cnitypes.Result
 	var err error
 	if delegate.ConfListPlugin {
-		result, err = conflistAdd(rt, delegate.Bytes, multusNetconf, exec)
+		// TODO: why are we passing bytes here? don't we have a better representation of it?
+		result, err = conflistAdd(rt, delegate.Bytes, &delegate.CNINetworkConfigList, multusNetconf, exec)
 		if err != nil {
 			return nil, err
 		}
@@ -369,11 +410,14 @@ func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 	}
 
 	if pod != nil {
-		// send kubernetes events
-		if delegate.Name != "" {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
-		} else {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+		// check Interfaces and IPs because some CNI plugin just return empty result
+		if res.Interfaces != nil || res.IPs != nil {
+			// send kubernetes events
+			if delegate.Name != "" {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
+			} else {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+			}
 		}
 	} else {
 		// for further debug https://github.com/k8snetworkplumbingwg/multus-cni/issues/481
@@ -508,7 +552,7 @@ func isCriticalRequestRetriable(err error) bool {
 
 // GetPod retrieves Kubernetes Pod object from given namespace/name in k8sArgs (i.e. cni args)
 // GetPod also get pod UID, but it is not used to retrieve, but it is used for double check
-func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
+func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, isDel bool) (*v1.Pod, error) {
 	if kubeClient == nil {
 		return nil, nil
 	}
@@ -517,31 +561,67 @@ func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (
 	podName := string(k8sArgs.K8S_POD_NAME)
 	podUID := string(k8sArgs.K8S_POD_UID)
 
-	pod, err := kubeClient.GetPod(podNamespace, podName)
-	if err != nil {
-		// in case of a retriable error, retry 10 times with 0.25 sec interval
-		if isCriticalRequestRetriable(err) {
-			waitErr := wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
-				pod, err = kubeClient.GetPod(podNamespace, podName)
-				return pod != nil, err
-			})
-			// retry failed, then return error with retry out
-			if waitErr != nil {
-				return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
+	// Keep track of how long getting the pod takes
+	logging.Debugf("GetPod for [%s/%s] starting", podNamespace, podName)
+	start := time.Now()
+	defer func() {
+		logging.Debugf("GetPod for [%s/%s] took %v", podNamespace, podName, time.Since(start))
+	}()
+
+	// Use a fairly long 0.25 sec interval so we don't hammer the apiserver
+	pollDuration := shortPollDuration
+	retryOnNotFound := func(error) bool {
+		return false
+	}
+
+	if kubeClient.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", podNamespace, podName)
+		// Use short retry intervals with the informer since it's a local cache
+		pollDuration = informerPollDuration
+		// Retry NotFound on ADD since the cache may be a bit behind the apiserver
+		retryOnNotFound = func(e error) bool {
+			return !isDel && errors.IsNotFound(e)
+		}
+	}
+
+	var pod *v1.Pod
+	if err := wait.PollImmediate(pollDuration, shortPollTimeout, func() (bool, error) {
+		var getErr error
+		// Use context with a short timeout so the call to API server doesn't take too long.
+		ctx, cancel := context.WithTimeout(context.TODO(), pollDuration)
+		defer cancel()
+		pod, getErr = kubeClient.GetPodContext(ctx, podNamespace, podName)
+		if isCriticalRequestRetriable(getErr) || retryOnNotFound(getErr) {
+			return false, nil
+		}
+		return pod != nil, getErr
+	}); err != nil {
+		if errors.IsNotFound(err) {
+			// When pods are not found, this is "OK", it's a known condition for rapidly deleted pods, we'll just warn on it.
+			if !isDel {
+				logging.Verbosef("Warning: GetPod for [%s/%s] resulted in pod not found during CNI ADD (pod may have already been deleted): %v", podNamespace, podName, err)
 			}
-		} else if warnOnly && errors.IsNotFound(err) {
-			// If not found, proceed to remove interface with cache
-			return nil, nil
-		} else {
-			// Other case, return error
-			return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
+			return nil, errPodNotFound
+		}
+		// Try one more time to get the pod directly from the apiserver;
+		// TODO: figure out why static pods don't show up via the informer
+		// and always hit this case.
+		ctx, cancel := context.WithTimeout(context.TODO(), pollDuration)
+		defer cancel()
+		pod, err = kubeClient.GetPodAPILiveQuery(ctx, podNamespace, podName)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				logging.Verbosef("Warning: On live query retry, [%s/%s] pod not found during CNI ADD (pod may have already been deleted): %v", podNamespace, podName, err)
+				return nil, errPodNotFound
+			}
+			return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
 		}
 	}
 
 	// In case of static pod, UID through kube api is different because of mirror pod, hence it is expected.
 	if podUID != "" && string(pod.UID) != podUID && !k8s.IsStaticPod(pod) {
 		msg := fmt.Sprintf("expected pod UID %q but got %q from Kube API", podUID, pod.UID)
-		if warnOnly {
+		if isDel {
 			// On CNI DEL we just operate on the cache when these mismatch, we don't error out.
 			// For example: stateful sets namespace/name can remain the same while podUID changes.
 			logging.Verbosef("warning: %s", msg)
@@ -572,17 +652,18 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	}
 
 	if n.ReadinessIndicatorFile != "" {
-		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-			_, err := os.Stat(n.ReadinessIndicatorFile)
-			return err == nil, nil
-		})
-		if err != nil {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
 			return nil, cmdErr(k8sArgs, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
 		}
 	}
 
 	pod, err := GetPod(kubeClient, k8sArgs, false)
 	if err != nil {
+		if err == errPodNotFound {
+			emptyresult := emptyCNIResult(args, "1.0.0")
+			logging.Verbosef("CmdAdd: Warning: pod [%s/%s] not found, exiting with empty CNI result: %v", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, emptyresult)
+			return emptyresult, nil
+		}
 		return nil, err
 	}
 
@@ -604,6 +685,36 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		return nil, cmdErr(k8sArgs, "error loading k8s delegates k8s args: %v", err)
 	}
 
+	// we add to the auxiliary CNI chain here.
+	if n.AuxiliaryCNIChainName != "" {
+		logging.Debugf("Using AuxiliaryCNIChainName: %v", n.AuxiliaryCNIChainName)
+
+		// create an passthru cni conflist configuration with our aux chain cni chain name.
+		jsonString := fmt.Sprintf(`{"cniVersion":"%s","name":"%s","plugins":[{"type":"passthru","name":"passthru-cni"}]}`, n.CNIVersion, n.AuxiliaryCNIChainName)
+
+		// Convert the JSON string to a byte array
+		byteArray := []byte(jsonString)
+
+		// Let's try to get the cni path from the ClusterNetwork
+		if !strings.Contains(n.ClusterNetwork, "/") {
+			return nil, cmdErr(k8sArgs, "auxiliary chain used but ClusterNetwork must be a path, and it is not a path: %v", n.ClusterNetwork)
+		}
+
+		// Get the directory part of the ClusterNetwork path
+		// TODO: This could probably be improved.
+		cniPath := filepath.Dir(n.ClusterNetwork)
+
+		// Load chained delegates
+		delegate := k8s.LoadChainedDelegatesFromBytes(byteArray, cniPath)
+		if delegate != nil {
+			// Only if additional plugins were listed do we add this aux chain delegate.
+			if len(delegate.ConfList.Plugins) > 1 {
+				// Add the resulting delegate to n.Delegates
+				n.Delegates = append(n.Delegates, delegate)
+			}
+		}
+	}
+
 	// cache the multus config
 	if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
 		return nil, cmdErr(k8sArgs, "error saving the delegates: %v", err)
@@ -636,71 +747,80 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
-		// Remove gateway from routing table if the gateway is not used
-		deleteV4gateway := false
-		deleteV6gateway := false
-		adddefaultgateway := false
-		if delegate.IsFilterV4Gateway {
-			deleteV4gateway = true
-			logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
-		} else {
-			// Otherwise, determine if this interface now gets our default route.
-			// According to
-			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
-			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
-			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
-				deleteV4gateway = true
-				adddefaultgateway = true
-				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
-			}
-		}
-
-		if delegate.IsFilterV6Gateway {
-			deleteV6gateway = true
-			logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
-		} else {
-			// Otherwise, determine if this interface now gets our default route.
-			// According to
-			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
-			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
-			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
-				deleteV6gateway = true
-				adddefaultgateway = true
-				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
-			}
-		}
-
-		// Remove gateway if `default-route` network selection is specified
-		if deleteV4gateway || deleteV6gateway {
-			err = netutils.DeleteDefaultGW(args.Netns, ifName)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
-			}
-			err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
-			}
-		}
-
-		// Here we'll set the default gateway which specified in `default-route` network selection
-		if adddefaultgateway {
-			err = netutils.SetDefaultGW(args.Netns, ifName, *delegate.GatewayRequest)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
-			}
-			err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
-			}
-		}
-
 		// Master plugin result is always used if present
 		if delegate.MasterPlugin || result == nil {
 			result = tmpResult
 		}
 
+		res, err := cni100.NewResultFromResult(tmpResult)
+		if err != nil {
+			logging.Errorf("CmdAdd: failed to read result: %v, but proceed", err)
+		}
+
+		// check Interfaces and IPs because some CNI plugin does not create any interface
+		// and just returns empty result
+		if res != nil && (res.Interfaces != nil || res.IPs != nil) {
+			// Remove gateway from routing table if the gateway is not used
+			deleteV4gateway := false
+			deleteV6gateway := false
+			adddefaultgateway := false
+			if delegate.IsFilterV4Gateway {
+				deleteV4gateway = true
+				logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV4gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			if delegate.IsFilterV6Gateway {
+				deleteV6gateway = true
+				logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV6gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			// Remove gateway if `default-route` network selection is specified
+			if deleteV4gateway || deleteV6gateway {
+				err = netutils.DeleteDefaultGW(args.Netns, ifName)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
+				}
+				err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
+				}
+			}
+
+			// Here we'll set the default gateway which specified in `default-route` network selection
+			if adddefaultgateway {
+				err = netutils.SetDefaultGW(args.Netns, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
+				}
+				err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
+				}
+			}
+		}
+
 		// Read devInfo from CNIDeviceInfoFile if it exists so
 		// it can be copied to the NetworkStatus.
 		devinfo, err := getDelegateDeviceInfo(delegate, rt)
@@ -710,15 +830,18 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			logging.Debugf("CmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
 		}
 
-		// create the network status, only in case Multus as kubeconfig
+		// Create the network statuses, only in case Multus has kubeconfig
 		if kubeClient != nil && kc != nil {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
-				delegateNetStatus, err := nadutils.CreateNetworkStatus(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
+				delegateNetStatuses, err := nadutils.CreateNetworkStatuses(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
 				if err != nil {
-					return nil, cmdErr(k8sArgs, "error setting network status: %v", err)
+					return nil, cmdErr(k8sArgs, "error setting network statuses: %v", err)
 				}
 
-				netStatus = append(netStatus, *delegateNetStatus)
+				// Append all returned statuses after dereferencing each
+				for _, status := range delegateNetStatuses {
+					netStatus = append(netStatus, *status)
+				}
 			}
 		} else if devinfo != nil {
 			// Warn that devinfo exists but could not add it to downwards API
@@ -726,15 +849,17 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 	}
 
-	// set the network status annotation in apiserver, only in case Multus as kubeconfig
+	// set the network status annotation in apiserver, only in case Multus has kubeconfig
 	if kubeClient != nil && kc != nil {
 		if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
 			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
-				if strings.Contains(err.Error(), "failed to query the pod") {
-					return nil, cmdErr(k8sArgs, "error setting the networks status, pod was already deleted: %v", err)
+				if strings.Contains(err.Error(), `pod "`) && strings.Contains(err.Error(), `" not found`) {
+					// Tolerate issues with writing the status due to pod deletion, and log them.
+					logging.Verbosef("warning: tolerated failure writing network status (pod not found): %v", err)
+				} else {
+					return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 				}
-				return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 			}
 		}
 	}
@@ -776,21 +901,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return err
 	}
 
-	skipStatusUpdate := false
 	netns, err := ns.GetNS(args.Netns)
-	if err != nil {
-		// if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
-		// so don't return an error if the device is already removed.
-		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
-		_, ok := err.(ns.NSPathNotExistErr)
-		skipStatusUpdate = true
-		if ok {
-			logging.Debugf("CmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
-		} else {
-			logging.Debugf("CmdDel: WARNING failed to open netns %q: %v", netns, err)
-		}
-	}
-
 	if netns != nil {
 		defer netns.Close()
 	}
@@ -801,12 +912,13 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 	}
 
 	if in.ReadinessIndicatorFile != "" {
-		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-			_, err := os.Stat(in.ReadinessIndicatorFile)
-			return err == nil, nil
-		})
+		readinessfileexists, err := types.ReadinessIndicatorExistsNow(in.ReadinessIndicatorFile)
 		if err != nil {
-			return cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile (on del): %v", err)
+			return cmdErr(k8sArgs, "error checking readinessindicatorfile on CNI DEL @ %v: %v", in.ReadinessIndicatorFile, err)
+		}
+
+		if !readinessfileexists {
+			logging.Verbosef("warning: readinessindicatorfile @ %v does not exist on CNI DEL", in.ReadinessIndicatorFile)
 		}
 	}
 
@@ -819,8 +931,6 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 	if err != nil {
 		// GetPod may be failed but just do print error in its log and continue to delete
 		logging.Errorf("Multus: GetPod failed: %v, but continue to delete", err)
-		// skip status update because k8s api seems to be stucked
-		skipStatusUpdate = true
 	}
 
 	// Read the cache to get delegates json for the pod
@@ -885,21 +995,6 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
-	if kubeClient != nil {
-		if !skipStatusUpdate {
-			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
-				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
-				if err != nil {
-					// error happen but continue to delete
-					logging.Errorf("Multus: error unsetting the networks status: %v", err)
-				}
-			}
-		} else {
-			logging.Debugf("WARNING: Unset SetNetworkStatus skipped")
-		}
-	}
-
 	e := delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
 
 	// Enable Option only delegate plugin delete success to delete cache file
@@ -924,3 +1019,125 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 
 	return e
 }
+
+// CmdStatus ...
+func CmdStatus(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	n, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("CmdStatus: %v, %v, %v", args, exec, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error loading netconf: %v", err)
+	}
+
+	kubeClient, err = k8s.GetK8sClient(n.Kubeconfig, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error getting k8s client: %v", err)
+	}
+
+	if n.ReadinessIndicatorFile != "" {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
+			return cmdErr(nil, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
+		}
+	}
+
+	if n.ClusterNetwork != "" {
+		_, err = k8s.GetDefaultNetworks(nil, n, kubeClient, nil)
+		if err != nil {
+			return cmdErr(nil, "failed to get clusterNetwork: %v", err)
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	// invoke delegate's STATUS command
+	// we only need to check cluster network status
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{n.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
+
+	conf, err := libcni.ConfListFromBytes(n.Delegates[0].Bytes)
+	if err != nil {
+		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	}
+
+	err = cniNet.GetStatusNetworkList(context.TODO(), conf)
+	if err != nil {
+		return logging.Errorf("error in STATUS command: %v", err)
+	}
+
+	return nil
+}
+
+// CmdGC ...
+func CmdGC(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	n, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("CmdStatus: %v, %v, %v", args, exec, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error loading netconf: %v", err)
+	}
+
+	kubeClient, err = k8s.GetK8sClient(n.Kubeconfig, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error getting k8s client: %v", err)
+	}
+
+	if n.ReadinessIndicatorFile != "" {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
+			return cmdErr(nil, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
+		}
+	}
+
+	if n.ClusterNetwork != "" {
+		_, err = k8s.GetDefaultNetworks(nil, n, kubeClient, nil)
+		if err != nil {
+			return cmdErr(nil, "failed to get clusterNetwork: %v", err)
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	// invoke delegate's GC command
+	// we only need to check cluster network status
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{n.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
+
+	conf, err := libcni.ConfListFromBytes(n.Delegates[0].Bytes)
+	if err != nil {
+		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	}
+
+	validAttachments, err := gatherValidAttachmentsFromCache(n.CNIDir)
+	if err != nil {
+		return logging.Errorf("error in gather valid attachments: %v", err)
+	}
+
+	err = cniNet.GCNetworkList(context.TODO(), conf, &libcni.GCArgs{
+		ValidAttachments: validAttachments,
+	})
+	if err != nil {
+		return logging.Errorf("error in GC command: %v", err)
+	}
+
+	return nil
+}
+
+func emptyCNIResult(args *skel.CmdArgs, cniVersion string) *cni100.Result {
+	return &cni100.Result{
+		CNIVersion: cniVersion,
+		Interfaces: []*cni100.Interface{
+			{
+				Name:    args.IfName,
+				Sandbox: args.Netns,
+			},
+		},
+		IPs: []*cni100.IPConfig{
+			{
+				Address: net.IPNet{
+					IP:   net.ParseIP("0.0.0.0"),
+					Mask: net.CIDRMask(0, 32),
+				},
+				Gateway: net.ParseIP("0.0.0.0"),
+			},
+		},
+	}
+}

pkg/multus/multus_cni020_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -24,10 +26,8 @@ import (
 	types020 "github.com/containernetworking/cni/pkg/types/020"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 
@@ -41,20 +41,6 @@ var _ = Describe("multus operations", func() {
 		err := saveScratchNetConf("123456789", "", meme)
 		Expect(err).To(HaveOccurred())
 	})
-
-	It("fails to delete delegates with bad filepath", func() {
-		err := deleteDelegates("123456789", "bad!file!~?Path$^")
-		Expect(err).To(HaveOccurred())
-	})
-
-	It("delete delegates given good filepath", func() {
-		os.MkdirAll("/opt/cni/bin", 0755)
-		d1 := []byte("blah")
-		os.WriteFile("/opt/cni/bin/123456789", d1, 0644)
-
-		err := deleteDelegates("123456789", "/opt/cni/bin")
-		Expect(err).NotTo(HaveOccurred())
-	})
 })
 
 var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
@@ -99,7 +85,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -154,6 +140,67 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("executes delegates given faulty namespace", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -162,7 +209,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -216,7 +263,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -283,7 +330,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -329,7 +376,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -706,66 +753,4 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
-		args := &skel.CmdArgs{
-			ContainerID: "123456789",
-			Netns:       testNS.Path(),
-			IfName:      "eth0",
-			StdinData: []byte(`{
-	    "name": "node-cni-network",
-	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
-	    "defaultnetworkwaitseconds": 3,
-	    "delegates": [{
-	        "name": "weave1",
-	        "cniVersion": "0.2.0",
-	        "type": "weave-net"
-	    },{
-	        "name": "other1",
-	        "cniVersion": "0.2.0",
-	        "type": "other-plugin"
-	    }]
-	}`),
-		}
-
-		fExec := newFakeExec()
-		expectedResult1 := &types020.Result{
-			CNIVersion: "0.2.0",
-			IP4: &types020.IPConfig{
-				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-		}
-		expectedConf1 := `{
-	    "name": "weave1",
-	    "cniVersion": "0.2.0",
-	    "type": "weave-net"
-	}`
-		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
-
-		expectedResult2 := &types020.Result{
-			CNIVersion: "0.2.0",
-			IP4: &types020.IPConfig{
-				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "0.2.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
-
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
-
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
-		Expect(err).To(HaveOccurred())
-	})
-
 })

pkg/multus/multus_cni040_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -24,10 +26,8 @@ import (
 	cni040 "github.com/containernetworking/cni/pkg/types/040"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -78,7 +78,7 @@ var _ = Describe("multus operations cniVersion 0.3.1 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -247,6 +247,67 @@ var _ = Describe("multus operations cniVersion 0.3.1 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.3.1",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.3.1",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni040.Result{
+			CNIVersion: "0.3.1",
+			IPs: []*cni040.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			}},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.3.1",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin040(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &cni040.Result{
+			CNIVersion: "0.3.1",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.3.1",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin040(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*cni040.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("fails when pod UID is provided and does not match Kube API pod UID", func() {
 		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
 		net1 := `{
@@ -590,7 +651,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -658,7 +719,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -714,7 +775,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -790,7 +851,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -859,7 +920,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -905,7 +966,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -1439,67 +1500,4 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
-		args := &skel.CmdArgs{
-			ContainerID: "123456789",
-			Netns:       testNS.Path(),
-			IfName:      "eth0",
-			StdinData: []byte(`{
-	    "name": "node-cni-network",
-	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
-	    "defaultnetworkwaitseconds": 3,
-	    "delegates": [{
-	        "name": "weave1",
-	        "cniVersion": "0.4.0",
-	        "type": "weave-net"
-	    },{
-	        "name": "other1",
-	        "cniVersion": "0.4.0",
-	        "type": "other-plugin"
-	    }]
-	}`),
-		}
-
-		fExec := newFakeExec()
-		expectedResult1 := &cni040.Result{
-			CNIVersion: "0.4.0",
-			IPs: []*cni040.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-			},
-		}
-		expectedConf1 := `{
-	    "name": "weave1",
-	    "cniVersion": "0.4.0",
-	    "type": "weave-net"
-	}`
-		fExec.addPlugin040(nil, "eth0", expectedConf1, expectedResult1, nil)
-
-		expectedResult2 := &cni040.Result{
-			CNIVersion: "0.4.0",
-			IPs: []*cni040.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-			},
-		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "0.4.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin040(nil, "net1", expectedConf2, expectedResult2, nil)
-
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
-
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
-		Expect(err).To(HaveOccurred())
-	})
 })

pkg/multus/multus_cni100_test.go

@@ -14,29 +14,91 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
+	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
+	"sync"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	kapi "k8s.io/api/core/v1"
+	informerfactory "k8s.io/client-go/informers"
+	v1coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+
+	netdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	netdefinformerv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1"
 )
 
+func newPodInformer(ctx context.Context, kclient kubernetes.Interface) cache.SharedIndexInformer {
+	informerFactory := informerfactory.NewSharedInformerFactory(kclient, 0*time.Second)
+
+	podInformer := informerFactory.InformerFor(&kapi.Pod{}, func(c kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return v1coreinformers.NewFilteredPodInformer(
+			c,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+			nil)
+	})
+
+	informerFactory.Start(ctx.Done())
+
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), podInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	return podInformer
+}
+
+func newNetDefInformer(ctx context.Context, client netdefclient.Interface) cache.SharedIndexInformer {
+	informerFactory := netdefinformer.NewSharedInformerFactory(client, 0*time.Second)
+
+	netdefInformer := informerFactory.InformerFor(&netdefv1.NetworkAttachmentDefinition{}, func(client netdefclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return netdefinformerv1.NewNetworkAttachmentDefinitionInformer(
+			client,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	})
+
+	informerFactory.Start(ctx.Done())
+
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), netdefInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	return netdefInformer
+}
+
 var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 	var testNS ns.NetNS
 	var tmpDir string
 	resultCNIVersion := "1.0.0"
 	configPath := "/tmp/foo.multus.conf"
+	var ctx context.Context
+	var cancel context.CancelFunc
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
@@ -52,9 +114,12 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		// Touch the default network file.
 		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 
+		ctx, cancel = context.WithCancel(context.TODO())
 	})
 
 	AfterEach(func() {
+		cancel()
+
 		// Cleanup default network file.
 		if _, errStat := os.Stat(configPath); errStat == nil {
 			errRemove := os.Remove(configPath)
@@ -76,7 +141,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -144,7 +209,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -192,6 +257,71 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(err).To(MatchError("[//:weave1]: error adding container to network \"weave1\": DelegateAdd: cannot set \"weave-net\" interface name to \"eth0\": validateIfName: no net namespace fsdadfad found: failed to Statfs \"fsdadfad\": no such file or directory"))
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "0.4.0",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdCheck(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -200,7 +330,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -276,7 +406,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -345,7 +475,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -391,7 +521,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -773,6 +903,116 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
+	It("executes clusterNetwork delegate with a shared informer", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		podInformer := newPodInformer(ctx, fKubeClient.Client)
+		netdefInformer := newNetDefInformer(ctx, fKubeClient.NetClient)
+		fKubeClient.SetK8sClientInformers(podInformer, netdefInformer)
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("executes clusterNetwork delegate with a shared informer if pod is not immediately found", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		podInformer := newPodInformer(ctx, fKubeClient.Client)
+		netdefInformer := newNetDefInformer(ctx, fKubeClient.NetClient)
+		fKubeClient.SetK8sClientInformers(podInformer, netdefInformer)
+
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			wg.Done()
+			time.Sleep(1 * time.Second)
+			fKubeClient.AddPod(fakePod)
+		}()
+		wg.Wait()
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("Verify the cache is created in dataDir", func() {
 		tmpCNIDir := tmpDir + "/cniData"
 		err := os.Mkdir(tmpCNIDir, 0777)
@@ -922,7 +1162,47 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
+})
+
+var _ = Describe("multus operations cniVersion 1.1.0 config", func() {
+	var testNS ns.NetNS
+	var tmpDir string
+	configPath := "/tmp/foo.multus.conf"
+	var cancel context.CancelFunc
+
+	BeforeEach(func() {
+		// Create a new NetNS so we don't modify the host
+		var err error
+		testNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+		os.Setenv("CNI_NETNS", testNS.Path())
+		os.Setenv("CNI_PATH", "/some/path")
+
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		// Touch the default network file.
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+		_, cancel = context.WithCancel(context.TODO())
+	})
+
+	AfterEach(func() {
+		cancel()
+
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+		Expect(testNS.Close()).To(Succeed())
+		os.Unsetenv("CNI_PATH")
+		os.Unsetenv("CNI_ARGS")
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes delegates with CNI Check", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -934,55 +1214,112 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
-	        "cniVersion": "1.0.0",
-	        "type": "weave-net"
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "weave-net"
+	        }]
 	    },{
 	        "name": "other1",
-	        "cniVersion": "1.0.0",
-	        "type": "other-plugin"
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "other-plugin"
+		}]
 	    }]
 	}`),
 		}
 
+		logging.SetLogLevel("verbose")
+
 		fExec := newFakeExec()
-		expectedResult1 := &cni100.Result{
-			CNIVersion: "1.0.0",
-			IPs: []*cni100.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-			},
-		}
 		expectedConf1 := `{
 	    "name": "weave1",
-	    "cniVersion": "1.0.0",
+	    "cniVersion": "1.1.0",
 	    "type": "weave-net"
 	}`
-		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "", expectedConf1, nil, nil)
 
-		expectedResult2 := &cni100.Result{
-			CNIVersion: "1.0.0",
-			IPs: []*cni100.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-			},
+		err := CmdStatus(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		// we only execute once for cluster network, not additional one
+		Expect(fExec.statusIndex).To(Equal(1))
+	})
+
+	It("executes delegates with CNI GC", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		cniCacheDir := filepath.Join(tmpCNIDir, "/results")
+		err = os.Mkdir(cniCacheDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		//create fake cniResult file
+		err = os.WriteFile(filepath.Join(cniCacheDir, "cbr0-3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755-eth0"), []byte(`{"kind":"cniCacheV1","containerId":"3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755","config":"eyJjbmlWZXJzaW9uIjoiMC4zLjEiLCJuYW1lIjoiY2JyMCIsInBsdWdpbnMiOlt7ImNhcGFiaWxpdGllcyI6eyJpby5rdWJlcm5ldGVzLmNyaS5wb2QtYW5ub3RhdGlvbnMiOnRydWV9LCJkZWxlZ2F0ZSI6eyJoYWlycGluTW9kZSI6dHJ1ZSwiaXNEZWZhdWx0R2F0ZXdheSI6dHJ1ZX0sInR5cGUiOiJmbGFubmVsIn0seyJjYXBhYmlsaXRpZXMiOnsicG9ydE1hcHBpbmdzIjp0cnVlfSwidHlwZSI6InBvcnRtYXAifV19","ifName":"eth0","networkName":"cbr0","netns":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e","cniArgs":[["IgnoreUnknown","true"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"],["IgnoreUnknown","1"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"]],"result":{"cniVersion":"0.3.1","dns":{},"interfaces":[{"mac":"ea:19:25:a2:a1:93","name":"cni0"},{"mac":"ba:76:61:2f:8b:ca","name":"vethc42d3d18"},{"mac":"7e:57:6a:9b:6b:b5","name":"eth0","sandbox":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e"}],"ips":[{"address":"10.244.1.4/24","gateway":"10.244.1.1","interface":2,"version":"4"}],"routes":[{"dst":"10.244.0.0/16"},{"dst":"0.0.0.0/0","gw":"10.244.1.1"}]}}`), 0666)
+		Expect(err).NotTo(HaveOccurred())
+		err = os.WriteFile(filepath.Join(cniCacheDir, "macvlan-conf-1-3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755-net1"), []byte(`{"kind":"cniCacheV1","containerId":"3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755","config":"eyJjbmlWZXJzaW9uIjoiMC4zLjEiLCJpcGFtIjp7ImFkZHJlc3NlcyI6W3siYWRkcmVzcyI6IjEwLjEuMS4xMDEvMjQifV0sInR5cGUiOiJzdGF0aWMifSwibWFzdGVyIjoiZXRoMSIsIm1vZGUiOiJicmlkZ2UiLCJuYW1lIjoibWFjdmxhbi1jb25mLTEiLCJ0eXBlIjoibWFjdmxhbiJ9","ifName":"net1","networkName":"macvlan-conf-1","netns":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e","cniArgs":[["IgnoreUnknown","true"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"],["IgnoreUnknown","1"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"]],"result":{"cniVersion":"0.3.1","dns":{},"interfaces":[{"mac":"36:b3:c5:29:ad:b8","name":"net1","sandbox":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e"}],"ips":[{"address":"10.1.1.101/24","interface":0,"version":"4"}]}}`), 0666)
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "weave-net"
+	        }]
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "other-plugin"
+		}]
+	    }]
+	}`, tmpCNIDir)),
 		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "1.0.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
 
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"1.0.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
+		logging.SetLogLevel("verbose")
 
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
-		Expect(err).To(HaveOccurred())
+		fExec := newFakeExec()
+		expectedConf1 := `{
+			"cni.dev/attachments": [
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "eth0"
+				},
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "net1"
+				}
+			],
+			"cni.dev/valid-attachments": [
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "eth0"
+				},
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "net1"
+				}
+			],
+			"cniVersion": "1.1.0",
+			"name": "weave1",
+			"type": "weave-net"
+		}`
+		fExec.addPlugin100(nil, "", expectedConf1, nil, nil)
+
+		err = CmdGC(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		// we only execute once for cluster network, not additional one
+		Expect(fExec.gcIndex).To(Equal(1))
+		err = os.RemoveAll(tmpCNIDir)
+		Expect(err).NotTo(HaveOccurred())
 	})
 })

pkg/multus/multus_suite_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"bytes"
 	"context"
@@ -56,6 +58,8 @@ type fakeExec struct {
 	addIndex        int
 	delIndex        int
 	chkIndex        int
+	statusIndex     int
+	gcIndex         int
 	expectedDelSkip int
 	plugins         map[string]*fakePlugin
 }
@@ -166,6 +170,14 @@ func (f *fakeExec) ExecPlugin(_ context.Context, pluginPath string, stdinData []
 		Expect(len(f.plugins)).To(BeNumerically(">", f.delIndex))
 		index = len(f.plugins) - f.expectedDelSkip - f.delIndex - 1
 		f.delIndex++
+	case "GC":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.statusIndex))
+		index = f.gcIndex
+		f.gcIndex++
+	case "STATUS":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.statusIndex))
+		index = f.statusIndex
+		f.statusIndex++
 	default:
 		// Should never be reached
 		Expect(false).To(BeTrue())
@@ -224,7 +236,7 @@ func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) {
 func NewFakeClientInfo() *k8sclient.ClientInfo {
 	return &k8sclient.ClientInfo{
 		Client:        fake.NewSimpleClientset(),
-		NetClient:     netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient:     netfake.NewSimpleClientset(),
 		EventRecorder: record.NewFakeRecorder(10),
 	}
 }

pkg/netutils/netutils.go

@@ -139,6 +139,7 @@ func deleteDefaultGWCacheBytes(cacheFile []byte, ipv4, ipv6 bool) ([]byte, error
 }
 
 func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interface{}, error) {
+	var newRoutes []interface{}
 	for i, r := range routes {
 		route, ok := r.(map[string]interface{})
 		if !ok {
@@ -150,12 +151,12 @@ func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interfac
 			if !ok {
 				return nil, fmt.Errorf("wrong dst format: %v", route["dst"])
 			}
-			if dst == dstGW {
-				routes = append(routes[:i], routes[i+1:]...)
+			if dst != dstGW {
+				newRoutes = append(newRoutes, routes[i])
 			}
 		}
 	}
-	return routes, nil
+	return newRoutes, nil
 }
 
 func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
@@ -205,7 +206,12 @@ func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[
 			return nil, err
 		}
 	}
-	result["routes"] = routes
+
+	if len(routes) == 0 {
+		delete(result, "routes")
+	} else {
+		result["routes"] = routes
+	}
 
 	return result, nil
 }

pkg/netutils/netutils_test.go

@@ -15,6 +15,8 @@
 
 package netutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"net"
@@ -639,6 +641,39 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(5))
 		})
 
+		It("verify ipv4 default gateway from single routes is removed/added from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "0.0.0.0/0",
+        "gw": "10.1.1.1"
+      }
+    ]
+  }
+}`)
+			newResult1, err := deleteDefaultGWCacheBytes(origResult, true, false)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = addDefaultGWCacheBytes(newResult1, []net.IP{net.ParseIP("10.1.1.1")})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("verify ipv6 default gateway is removed from CNI 1.0.0 results", func() {
 			origResult := []byte(`{
   "kind": "cniCacheV1",
@@ -711,6 +746,39 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(5))
 		})
 
+		It("verify ipv6 default gateway from single routes is removed/added from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "::0/0",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult1, err := deleteDefaultGWCacheBytes(origResult, false, true)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = addDefaultGWCacheBytes(newResult1, []net.IP{net.ParseIP("10::1:1:1")})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("verify ipv4 default gateway is added to CNI 0.1.0/0.2.0 results without routes", func() {
 			origResult := []byte(`{
   "kind": "cniCacheV1",
@@ -1421,3 +1489,23 @@ var _ = Describe("netutil cnicache function testing", func() {
 
 	})
 })
+
+var _ = Describe("other function unit testing", func() {
+	It("deleteDefaultGWResultRoutes with invalid config", func() {
+		cniRouteConfig := []byte(`[
+		{ "dst": "0.0.0.0/0", "gw": "10.1.1.1" },
+		{ "dst": "10.1.1.0/24" },
+		{ "dst": "0.0.0.0/0", "gw": "10.1.1.1" }
+		]`)
+
+		var routes []interface{}
+		err := json.Unmarshal(cniRouteConfig, &routes)
+		Expect(err).NotTo(HaveOccurred())
+
+		newRoute, err := deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+		Expect(err).NotTo(HaveOccurred())
+		routeJSON, err := json.Marshal(newRoute)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(routeJSON).Should(MatchJSON(`[{"dst":"10.1.1.0/24"}]`))
+	})
+})

pkg/server/api/api.go

@@ -22,9 +22,17 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"time"
+
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
+	// APIReadyPollDuration specifies duration for API readiness check polling
+	APIReadyPollDuration = 100 * time.Millisecond
+	// APIReadyPollTimeout specifies timeout for API readiness check polling
+	APIReadyPollTimeout = 60000 * time.Millisecond
+
 	// MultusCNIAPIEndpoint is an endpoint for multus CNI request (for multus-shim)
 	MultusCNIAPIEndpoint = "/cni"
 	// MultusDelegateAPIEndpoint is an endpoint for multus delegate request (for hotplug)
@@ -45,7 +53,7 @@ func DoCNI(url string, req interface{}, socketPath string) ([]byte, error) {
 
 	client := &http.Client{
 		Transport: &http.Transport{
-			Dial: func(proto, addr string) (net.Conn, error) {
+			Dial: func(_, _ string) (net.Conn, error) {
 				return net.Dial("unix", socketPath)
 			},
 		},
@@ -88,3 +96,20 @@ func CreateDelegateRequest(cniCommand, cniContainerID, cniNetNS, cniIFName, podN
 		InterfaceAttributes: interfaceAttributes,
 	}
 }
+
+// WaitUntilAPIReady checks API readiness
+func WaitUntilAPIReady(socketPath string) error {
+	return utilwait.PollImmediate(APIReadyPollDuration, APIReadyPollTimeout, func() (bool, error) {
+		_, err := DoCNI(GetAPIEndpoint(MultusHealthAPIEndpoint), nil, SocketPath(socketPath))
+		return err == nil, nil
+	})
+}
+
+// CheckAPIReadyNow checks API readiness once
+func CheckAPIReadyNow(socketPath string) error {
+	_, err := DoCNI(GetAPIEndpoint(MultusHealthAPIEndpoint), nil, SocketPath(socketPath))
+	if err != nil {
+		return fmt.Errorf("CheckAPIReadyNow: Daemon not reachable over socketfile: %v", err)
+	}
+	return nil
+}

pkg/server/api/shim.go

@@ -40,9 +40,12 @@ type ShimNetConf struct {
 	LogToStderr     bool   `json:"logToStderr,omitempty"`
 }
 
+// readyCheckFunc defines a type for API readiness check functions
+type readyCheckFunc func(string) error
+
 // CmdAdd implements the CNI spec ADD command handler
 func CmdAdd(args *skel.CmdArgs) error {
-	response, cniVersion, err := postRequest(args)
+	response, cniVersion, err := postRequest(args, WaitUntilAPIReady)
 	if err != nil {
 		return logging.Errorf("CmdAdd (shim): %v", err)
 	}
@@ -53,7 +56,7 @@ func CmdAdd(args *skel.CmdArgs) error {
 
 // CmdCheck implements the CNI spec CHECK command handler
 func CmdCheck(args *skel.CmdArgs) error {
-	_, _, err := postRequest(args)
+	_, _, err := postRequest(args, WaitUntilAPIReady)
 	if err != nil {
 		return logging.Errorf("CmdCheck (shim): %v", err)
 	}
@@ -63,27 +66,52 @@ func CmdCheck(args *skel.CmdArgs) error {
 
 // CmdDel implements the CNI spec DEL command handler
 func CmdDel(args *skel.CmdArgs) error {
-	_, _, err := postRequest(args)
+	_, _, err := postRequest(args, CheckAPIReadyNow)
 	if err != nil {
-		return logging.Errorf("CmdDel (shim): %v", err)
+		// No error in DEL (as of CNI spec)
+		logging.Errorf("CmdDel (shim): %v", err)
 	}
 	return nil
 }
 
-func postRequest(args *skel.CmdArgs) (*Response, string, error) {
+// CmdGC implements the CNI spec GC command handler
+func CmdGC(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args, WaitUntilAPIReady)
+	if err != nil {
+		return logging.Errorf("CmdGC (shim): %v", err)
+	}
+	return nil
+}
+
+// CmdStatus implements the CNI spec STATUS command handler
+func CmdStatus(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args, WaitUntilAPIReady)
+	if err != nil {
+		return logging.Errorf("CmdStatus (shim): %v", err)
+	}
+	return nil
+}
+
+func postRequest(args *skel.CmdArgs, readinessCheck readyCheckFunc) (*Response, string, error) {
 	multusShimConfig, err := shimConfig(args.StdinData)
 	if err != nil {
 		return nil, "", fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err)
 	}
 
+	// Execute the readiness check as necessary (e.g. don't wait on CNI DEL)
+	if err := readinessCheck(multusShimConfig.MultusSocketDir); err != nil {
+		return nil, multusShimConfig.CNIVersion, err
+	}
+
 	cniRequest, err := newCNIRequest(args)
 	if err != nil {
 		return nil, multusShimConfig.CNIVersion, err
 	}
 
-	body, err := DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
+	var body []byte
+	body, err = DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
 	if err != nil {
-		return nil, multusShimConfig.CNIVersion, err
+		return nil, multusShimConfig.CNIVersion, fmt.Errorf("%s: StdinData: %s", err.Error(), string(args.StdinData))
 	}
 
 	response := &Response{}

pkg/server/config/config_suite_test.go

@@ -14,6 +14,8 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/server/config/generator.go

@@ -55,7 +55,8 @@ type MultusConf struct {
 	Type                     string              `json:"type"`
 	CniDir                   string              `json:"cniDir,omitempty"`
 	CniConfigDir             string              `json:"cniConfigDir,omitempty"`
-	SocketDir                string              `json:"socketDir,omitempty"`
+	AuxiliaryCNIChainName    string              `json:"auxiliaryCNIChainName,omitempty"`
+	DaemonSocketDir          string              `json:"daemonSocketDir,omitempty"`
 	MultusConfigFile         string              `json:"multusConfigFile,omitempty"`
 	MultusMasterCni          string              `json:"multusMasterCNI,omitempty"`
 	MultusAutoconfigDir      string              `json:"multusAutoconfigDir,omitempty"`
@@ -126,6 +127,11 @@ func (mc *MultusConf) Generate() (string, error) {
 	mc.CniConfigDir = ""
 	mc.MultusConfigFile = ""
 	mc.MultusAutoconfigDir = ""
+	mc.MultusMasterCni = ""
+	mc.ForceCNIVersion = false
+	// Readiness indicator file existence is already handled by the
+	// ConfigManager via an fsnotify watch, so CmdAdd/CmdDel don't need to.
+	mc.ReadinessIndicatorFile = ""
 
 	data, err := json.Marshal(mc)
 	return string(data), err

pkg/server/config/generator_test.go

@@ -14,6 +14,8 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"fmt"

pkg/server/config/manager.go

@@ -15,9 +15,12 @@
 package config
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"sync"
 
 	"github.com/fsnotify/fsnotify"
 
@@ -34,39 +37,43 @@ const (
 // Manager monitors the configuration of the primary CNI plugin, and
 // regenerates multus configuration whenever it gets updated.
 type Manager struct {
-	cniConfigData        map[string]interface{}
-	configWatcher        *fsnotify.Watcher
-	multusConfig         *MultusConf
-	multusConfigDir      string
-	multusConfigFilePath string
-	primaryCNIConfigPath string
+	cniConfigData              map[string]interface{}
+	configWatcher              *fsnotify.Watcher
+	multusConfig               *MultusConf
+	multusConfigDir            string
+	multusConfigFilePath       string
+	readinessIndicatorFilePath string
+	primaryCNIConfigPath       string
 }
 
 // NewManager returns a config manager object, configured to read the
-// primary CNI configuration in `multusAutoconfigDir`. This constructor will auto-discover
-// the primary CNI for which it will delegate.
-func NewManager(config MultusConf, multusAutoconfigDir string, forceCNIVersion bool) (*Manager, error) {
-	defaultCNIPluginName, err := getPrimaryCNIPluginName(multusAutoconfigDir)
-	if err != nil {
-		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
-		return nil, err
+// primary CNI configuration in `config.MultusAutoconfigDir`. If
+// `config.MultusMasterCni` is empty, this constructor will auto-discover the
+// primary CNI for which it will delegate.
+func NewManager(config MultusConf) (*Manager, error) {
+	var err error
+	defaultPluginName := config.MultusMasterCni
+	if defaultPluginName == "" {
+		defaultPluginName, err = getPrimaryCNIPluginName(config.MultusAutoconfigDir)
+		if err != nil {
+			_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+			return nil, err
+		}
 	}
-	return newManager(config, multusAutoconfigDir, defaultCNIPluginName, forceCNIVersion)
-}
 
-// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
-// configured to persist the configuration to `multusAutoconfigDir`. This
-// constructor will use the primary CNI plugin indicated by the user, via the
-// primaryCNIPluginName variable.
-func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir, primaryCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
-	return newManager(config, multusAutoconfigDir, primaryCNIPluginName, forceCNIVersion)
+	return newManager(config, defaultPluginName)
 }
 
 // overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case
 func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
-	masterCNIConfigData, err := os.ReadFile(cniConfigFile)
+	path, err := filepath.Abs(cniConfigFile)
 	if err != nil {
-		return fmt.Errorf("failed to read cni config %s: %v", cniConfigFile, err)
+		return fmt.Errorf("illegal path %s in cni config path %s: %w", path, cniConfigFile, err)
+	}
+
+	masterCNIConfigData, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read cni config %s: %v", path, err)
 	}
 
 	var primaryCNIConfigData map[string]interface{}
@@ -80,45 +87,85 @@ func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
 		return fmt.Errorf("couldn't update cluster network config: %v", err)
 	}
 
-	err = os.WriteFile(cniConfigFile, configBytes, 0644)
+	err = os.WriteFile(path, configBytes, 0644)
 	if err != nil {
 		return fmt.Errorf("couldn't update cluster network config: %v", err)
 	}
 	return nil
 }
 
-func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
-	if forceCNIVersion {
-		err := overrideCNIVersion(cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), config.CNIVersion)
+func newManager(config MultusConf, defaultCNIPluginName string) (*Manager, error) {
+	if config.ForceCNIVersion {
+		err := overrideCNIVersion(filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName), config.CNIVersion)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	watcher, err := newWatcher(multusConfigDir)
+	readinessIndicatorPath := ""
+	if config.ReadinessIndicatorFile != "" {
+		readinessIndicatorPath = filepath.Dir(config.ReadinessIndicatorFile)
+	}
+
+	watcher, err := newWatcher(config.MultusAutoconfigDir, readinessIndicatorPath)
 	if err != nil {
 		return nil, err
 	}
 
-	if defaultCNIPluginName == fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) {
-		return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", multusConfigDir, multusConfigFileName)
+	if defaultCNIPluginName == fmt.Sprintf("%s/%s", config.MultusAutoconfigDir, multusConfigFileName) {
+		return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", config.MultusAutoconfigDir, multusConfigFileName)
 	}
 
 	configManager := &Manager{
-		configWatcher:        watcher,
-		multusConfig:         &config,
-		multusConfigDir:      multusConfigDir,
-		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
-		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+		configWatcher:              watcher,
+		multusConfig:               &config,
+		multusConfigDir:            config.MultusAutoconfigDir,
+		multusConfigFilePath:       filepath.Join(config.CniConfigDir, multusConfigFileName),
+		primaryCNIConfigPath:       filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName),
+		readinessIndicatorFilePath: config.ReadinessIndicatorFile,
 	}
 
 	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
 		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
 	}
 
+	if config.OverrideNetworkName {
+		if err := configManager.overrideNetworkName(); err != nil {
+			return nil, logging.Errorf("could not override the network name: %v", err)
+		}
+	}
+
 	return configManager, nil
 }
 
+// Start generates an updated Multus config, writes it, and begins watching
+// the config directory and readiness indicator files for changes
+func (m *Manager) Start(ctx context.Context, wg *sync.WaitGroup) error {
+	generatedMultusConfig, err := m.GenerateConfig()
+	if err != nil {
+		return logging.Errorf("failed to generated the multus configuration: %v", err)
+	}
+	logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+	multusConfigFile, err := m.PersistMultusConfig(generatedMultusConfig)
+	if err != nil {
+		return logging.Errorf("failed to persist the multus configuration: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := m.monitorPluginConfiguration(ctx); err != nil {
+			_ = logging.Errorf("error watching file: %v", err)
+		}
+		logging.Verbosef("ConfigWatcher done")
+		logging.Verbosef("Delete old config @ %v", multusConfigFile)
+		os.Remove(multusConfigFile)
+	}()
+
+	return nil
+}
+
 func (m *Manager) loadPrimaryCNIConfigFromFile() error {
 	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
 	if err != nil {
@@ -132,9 +179,9 @@ func (m *Manager) loadPrimaryCNIConfigFromFile() error {
 	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
 }
 
-// OverrideNetworkName overrides the name of the multus configuration with the
+// overrideNetworkName overrides the name of the multus configuration with the
 // name of the delegated primary CNI.
-func (m *Manager) OverrideNetworkName() error {
+func (m *Manager) overrideNetworkName() error {
 	name, ok := m.cniConfigData["name"]
 	if !ok {
 		return fmt.Errorf("failed to access delegate CNI plugin name")
@@ -157,7 +204,7 @@ func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface
 }
 
 // GenerateConfig generates a multus configuration from its current state
-func (m Manager) GenerateConfig() (string, error) {
+func (m *Manager) GenerateConfig() (string, error) {
 	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
 		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
 		return "", nil
@@ -165,25 +212,25 @@ func (m Manager) GenerateConfig() (string, error) {
 	return m.multusConfig.Generate()
 }
 
-// MonitorPluginConfiguration monitors the configuration file pointed
+// monitorPluginConfiguration monitors the configuration file pointed
 // to by the primaryCNIPluginName attribute, and re-generates the multus
 // configuration whenever the primary CNI config is updated.
-func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<- struct{}) error {
+func (m *Manager) monitorPluginConfiguration(ctx context.Context) error {
 	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
 
 	for {
 		select {
 		case event := <-m.configWatcher.Events:
-			// we're watching the DIR where the config sits, and the event
-			// does not concern the primary CNI config. Skip it.
-			if event.Name != m.primaryCNIConfigPath {
-				logging.Debugf("skipping un-related event %v", event)
+			if !m.shouldRegenerateConfig(event) {
 				continue
 			}
 			logging.Debugf("process event: %v", event)
 
-			if !shouldRegenerateConfig(event) {
-				continue
+			// if readinessIndicatorFile is removed, then restart multus
+			if m.readinessIndicatorFilePath != "" && m.readinessIndicatorFilePath == event.Name {
+				logging.Verbosef("readiness indicator file is gone. restart multus-daemon")
+				os.Remove(m.multusConfigFilePath)
+				os.Exit(2)
 			}
 
 			updatedConfig, err := m.GenerateConfig()
@@ -192,7 +239,7 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 			}
 
 			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
-			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+			if _, err := m.PersistMultusConfig(updatedConfig); err != nil {
 				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
 			}
 			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
@@ -205,10 +252,9 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 			}
 			logging.Errorf("CNI monitoring error %v", err)
 
-		case <-shutDown:
+		case <-ctx.Done():
 			logging.Verbosef("Stopped monitoring, closing channel ...")
 			_ = m.configWatcher.Close()
-			close(done)
 			return nil
 		}
 	}
@@ -216,9 +262,28 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 
 // PersistMultusConfig persists the provided configuration to the disc, with
 // Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
-func (m Manager) PersistMultusConfig(config string) error {
-	logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
-	return os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
+func (m *Manager) PersistMultusConfig(config string) (string, error) {
+	if _, err := os.Stat(m.multusConfigFilePath); err == nil {
+		logging.Debugf("Overwriting Multus CNI configuration @ %s", m.multusConfigFilePath)
+	} else {
+		logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
+	}
+	return m.multusConfigFilePath, os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
+}
+
+func (m *Manager) shouldRegenerateConfig(event fsnotify.Event) bool {
+	// first, check the readiness indicator file existence
+	if event.Name == m.readinessIndicatorFilePath {
+		return event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename)
+	}
+
+	// we're watching the DIR where the config sits, and the event
+	// does not concern the primary CNI config. Skip it.
+	if event.Name == m.primaryCNIConfigPath {
+		return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
+	}
+	logging.Debugf("skipping un-related event %v", event)
+	return false
 }
 
 func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
@@ -229,11 +294,7 @@ func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
 	return masterCniConfigFileName, nil
 }
 
-func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
-	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
-}
-
-func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+func newWatcher(cniConfigDir string, readinessIndicatorDir string) (*fsnotify.Watcher, error) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
@@ -246,16 +307,18 @@ func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
 	}()
 
 	if err = watcher.Add(cniConfigDir); err != nil {
-		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+		return nil, fmt.Errorf("failed to add watch on %q for cni config: %v", cniConfigDir, err)
+	}
+	// if readinessIndicatorDir is different from cniConfigDir,
+	if readinessIndicatorDir != "" && cniConfigDir != readinessIndicatorDir {
+		if err = watcher.Add(readinessIndicatorDir); err != nil {
+			return nil, fmt.Errorf("failed to add watch on %q for readinessIndicator: %v", readinessIndicatorDir, err)
+		}
 	}
 
 	return watcher, nil
 }
 
-func shouldRegenerateConfig(event fsnotify.Event) bool {
-	return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
-}
-
 func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
 	masterCNIConfigData, err := os.ReadFile(masterCNIPluginPath)
 	if err != nil {

pkg/server/config/manager_test.go

@@ -14,11 +14,14 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
-	"time"
+	"sync"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -41,6 +44,7 @@ var _ = Describe("Configuration Manager", func() {
 	var configManager *Manager
 	var multusConfigDir string
 	var defaultCniConfig string
+	var wg *sync.WaitGroup
 
 	BeforeEach(func() {
 		var err error
@@ -53,19 +57,25 @@ var _ = Describe("Configuration Manager", func() {
 
 		multusConfFile := fmt.Sprintf(`{
 			"name": %q,
-			"cniVersion": %q
-		}`, defaultCniConfig, cniVersion)
+			"cniVersion": %q,
+			"multusAutoconfigDir": %q,
+			"multusMasterCNI": %q,
+			"forceCNIVersion": false
+		}`, defaultCniConfig, cniVersion, multusConfigDir, primaryCNIPluginName)
 		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
 		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
 
 		multusConf, err := ParseMultusConfig(multusConfFileName)
 		Expect(err).NotTo(HaveOccurred())
 
-		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		configManager, err = NewManager(*multusConf)
 		Expect(err).NotTo(HaveOccurred())
+
+		wg = &sync.WaitGroup{}
 	})
 
 	AfterEach(func() {
+		wg.Wait()
 		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
 	})
 
@@ -95,23 +105,17 @@ var _ = Describe("Configuration Manager", func() {
 	})
 
 	It("Check MonitorPluginConfiguration", func() {
-		config, err := configManager.GenerateConfig()
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		err := configManager.Start(ctx, wg)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = configManager.PersistMultusConfig(config)
-		Expect(err).NotTo(HaveOccurred())
-
-		configWatcherDoneChannel := make(chan struct{})
-		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
-			err := configManager.MonitorPluginConfiguration(configWatcherDoneChannel, stopChannel)
-			Expect(err).NotTo(HaveOccurred())
-		}(make(chan struct{}), configWatcherDoneChannel)
-
 		updatedCNIConfig := `
 {
   "cniVersion": "0.4.0",
   "name": "mycni-name",
   "type": "mycni2",
+  "capabilities": {"portMappings": true},
   "ipam": {},
   "dns": {}
 }
@@ -120,18 +124,16 @@ var _ = Describe("Configuration Manager", func() {
 		Expect(os.WriteFile(defaultCniConfig, []byte(updatedCNIConfig), UserRWPermission)).To(Succeed())
 
 		// wait for a while to get fsnotify event
-		time.Sleep(100 * time.Millisecond)
-		file, err := os.ReadFile(configManager.multusConfigFilePath)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(string(file)).To(Equal(config))
-
-		// stop groutine
-		configWatcherDoneChannel <- struct{}{}
+		Eventually(func() string {
+			file, err := os.ReadFile(configManager.multusConfigFilePath)
+			Expect(err).NotTo(HaveOccurred())
+			return string(file)
+		}, 2).Should(ContainSubstring("portMappings"))
 	})
 
 	When("the user requests the name of the multus configuration to be overridden", func() {
 		BeforeEach(func() {
-			Expect(configManager.OverrideNetworkName()).To(Succeed())
+			Expect(configManager.overrideNetworkName()).To(Succeed())
 		})
 
 		It("Overrides the name of the multus configuration when requested", func() {
@@ -171,14 +173,17 @@ var _ = Describe("Configuration Manager with mismatched cniVersion", func() {
 
 		multusConfFile := fmt.Sprintf(`{
 			"name": %q,
-			"cniVersion": %q
-		}`, defaultCniConfig, cniVersion)
+			"cniVersion": %q,
+			"multusAutoconfigDir": %q,
+			"multusMasterCNI": %q,
+			"forceCNIVersion": false
+		}`, defaultCniConfig, cniVersion, multusConfigDir, primaryCNIPluginName)
 		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
 		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
 
 		multusConf, err := ParseMultusConfig(multusConfFileName)
 		Expect(err).NotTo(HaveOccurred())
-		_, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		_, err = NewManager(*multusConf)
 		Expect(err).To(MatchError("failed to load the primary CNI configuration as a multus delegate with error 'delegate cni version is 0.3.1 while top level cni version is 0.4.0'"))
 	})
 

pkg/server/exec_chroot.go

@@ -20,10 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"os/exec"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -34,91 +32,24 @@ import (
 
 // ChrootExec implements invoke.Exec to execute CNI with chroot
 type ChrootExec struct {
-	Stderr     io.Writer
-	chrootDir  string
-	workingDir string   // working directory in the outer root
-	outerRoot  *os.File // outer root directory
+	Stderr    io.Writer
+	chrootDir string
 	version.PluginDecoder
-	mu sync.Mutex
 }
 
 var _ invoke.Exec = &ChrootExec{}
 
-func (e *ChrootExec) chroot() error {
-	var err error
-	e.workingDir, err = os.Getwd()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
-		return fmt.Errorf("getwd before chroot failed: %v", err)
-	}
-
-	e.outerRoot, err = os.Open("/")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
-		return fmt.Errorf("getwd before chroot failed: %v", err)
-	}
-
-	if err := syscall.Chroot(e.chrootDir); err != nil {
-		fmt.Fprintf(os.Stderr, "chroot to %s failed: %v\n", e.chrootDir, err)
-		return fmt.Errorf("chroot to %s failed: %v", e.chrootDir, err)
-	}
-
-	if err := os.Chdir("/"); err != nil {
-		fmt.Fprintf(os.Stderr, "chdir to \"/\" failed: %v\n", err)
-		return fmt.Errorf("chdir to \"/\" failed: %v", err)
-	}
-
-	return nil
-}
-
-func (e *ChrootExec) escape() error {
-	if e.outerRoot == nil || e.workingDir == "" {
-		return nil
-	}
-
-	// change directory to outer root and close it
-	if err := syscall.Fchdir(int(e.outerRoot.Fd())); err != nil {
-		fmt.Fprintf(os.Stderr, "changing directory to outer root failed: %v\n", err)
-		return fmt.Errorf("changing directory to outer root failed: %v", err)
-	}
-
-	if err := e.outerRoot.Close(); err != nil {
-		fmt.Fprintf(os.Stderr, "closing outer root failed: %v\n", err)
-		return fmt.Errorf("closing outer root failed: %v", err)
-	}
-
-	// chroot to current directory aka "." being the outer root
-	if err := syscall.Chroot("."); err != nil {
-		fmt.Fprintf(os.Stderr, "chroot to current directory failed: %v\n", err)
-		return fmt.Errorf("chroot to current directory failed: %v", err)
-	}
-
-	if err := os.Chdir(e.workingDir); err != nil {
-		fmt.Fprintf(os.Stderr, "chdir to working directory failed: %v\n", err)
-		return fmt.Errorf("chdir to working directory failed: %v", err)
-	}
-	e.outerRoot = nil
-	e.workingDir = ""
-
-	return nil
-}
-
 // ExecPlugin executes CNI plugin with given environment/stdin data.
 func (e *ChrootExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
-	// lock and do chroot to execute plugin with host root
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	err := e.chroot()
-	defer e.escape()
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "ExecPlugin failed at chroot: %v\n", err)
-		return nil, fmt.Errorf("ExecPlugin failed at chroot: %v", err)
-	}
+	var err error
 
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}
 	c := exec.CommandContext(ctx, pluginPath)
+	// execute delegate CNI with host filesystem context.
+	c.SysProcAttr = &syscall.SysProcAttr{
+		Chroot: e.chrootDir,
+	}
 	c.Env = environ
 	c.Stdin = bytes.NewBuffer(stdinData)
 	c.Stdout = stdout
@@ -169,14 +100,5 @@ func (e *ChrootExec) pluginErr(err error, stdout, stderr []byte) error {
 
 // FindInPath try to find CNI plugin based on given path
 func (e *ChrootExec) FindInPath(plugin string, paths []string) (string, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	err := e.chroot()
-	defer e.escape()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "FindInPath failed at chroot: %v\n", err)
-		return "", fmt.Errorf("FindInPath failed at chroot: %v", err)
-	}
-
 	return invoke.FindInPath(plugin, paths)
 }

pkg/server/exec_chroot_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"os"
@@ -43,15 +45,4 @@ var _ = Describe("exec_chroot", func() {
 		_, err := chrootExec.ExecPlugin(context.Background(), "/bin/true", nil, nil)
 		Expect(err).To(HaveOccurred())
 	})
-
-	It("Call ChrootExec.FindInPath with dummy", func() {
-		chrootExec := &ChrootExec{
-			Stderr:    os.Stderr,
-			chrootDir: "/usr/bin",
-		}
-
-		_, err := chrootExec.FindInPath("true", []string{"/"})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
 })

pkg/server/server.go

@@ -15,7 +15,7 @@
 package server
 
 import (
-	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -38,6 +39,24 @@ import (
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+
+	netdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	netdefinformerv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1"
+
+	kapi "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+	informerfactory "k8s.io/client-go/informers"
+	v1coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/informers/internalinterfaces"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
 )
 
 const (
@@ -57,61 +76,60 @@ func FilesystemPreRequirements(rundir string) error {
 	return nil
 }
 
+func printCmdArgs(args *skel.CmdArgs) string {
+	return fmt.Sprintf("ContainerID:%q Netns:%q IfName:%q Args:%q Path:%q",
+		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
+}
+
 // HandleCNIRequest is the CNI server handler function; it is invoked whenever
 // a CNI request is processed.
-func (s *Server) HandleCNIRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+func (s *Server) HandleCNIRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs) ([]byte, error) {
 	var result []byte
 	var err error
 
-	logging.Verbosef("%s starting CNI request %+v", cmd, cniCmdArgs)
+	logging.Verbosef("%s starting CNI request %s", cmd, printCmdArgs(cniCmdArgs))
 	switch cmd {
 	case "ADD":
-		result, err = cmdAdd(cniCmdArgs, k8sArgs, exec, kubeClient)
+		result, err = s.cmdAdd(cniCmdArgs, k8sArgs)
 	case "DEL":
-		err = cmdDel(cniCmdArgs, k8sArgs, exec, kubeClient)
+		err = s.cmdDel(cniCmdArgs, k8sArgs)
 	case "CHECK":
-		err = cmdCheck(cniCmdArgs, k8sArgs, exec, kubeClient)
+		err = s.cmdCheck(cniCmdArgs, k8sArgs)
+	case "GC":
+		err = s.cmdGC(cniCmdArgs, k8sArgs)
+	case "STATUS":
+		err = s.cmdStatus(cniCmdArgs, k8sArgs)
 	default:
 		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
 	}
-	logging.Verbosef("%s finished CNI request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
-	if err != nil {
-		// Prefix errors with request info for easier failure debugging
-		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
-	}
-	return result, nil
+	logging.Verbosef("%s finished CNI request %s, result: %q, err: %v", cmd, printCmdArgs(cniCmdArgs), string(result), err)
+	return result, err
 }
 
 // HandleDelegateRequest is the CNI server handler function; it is invoked whenever
 // a CNI request is processed as delegate CNI request.
-func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
 	var result []byte
 	var err error
-	var multusConfByte []byte
 
-	multusConfByte = bytes.Replace(s.serverConfig, []byte(","), []byte("{"), 1)
 	multusConfig := types.GetDefaultNetConf()
-	if err = json.Unmarshal(multusConfByte, multusConfig); err != nil {
+	if err = json.Unmarshal(s.serverConfig, multusConfig); err != nil {
 		return nil, err
 	}
 
-	logging.Verbosef("%s starting delegate request %+v", cmd, cniCmdArgs)
+	logging.Verbosef("%s starting delegate request %s", cmd, printCmdArgs(cniCmdArgs))
 	switch cmd {
 	case "ADD":
-		result, err = cmdDelegateAdd(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig, interfaceAttributes)
+		result, err = s.cmdDelegateAdd(cniCmdArgs, k8sArgs, multusConfig, interfaceAttributes)
 	case "DEL":
-		err = cmdDelegateDel(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+		err = s.cmdDelegateDel(cniCmdArgs, k8sArgs, multusConfig)
 	case "CHECK":
-		err = cmdDelegateCheck(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+		err = s.cmdDelegateCheck(cniCmdArgs, k8sArgs, multusConfig)
 	default:
 		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
 	}
-	logging.Verbosef("%s finished Delegate request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
-	if err != nil {
-		// Prefix errors with request info for easier failure debugging
-		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
-	}
-	return result, nil
+	logging.Verbosef("%s finished Delegate request %s, result: %q, err: %v", cmd, printCmdArgs(cniCmdArgs), string(result), err)
+	return result, err
 }
 
 // GetListener creates a listener to a unix socket located in `socketPath`
@@ -127,11 +145,104 @@ func GetListener(socketPath string) (net.Listener, error) {
 	return l, nil
 }
 
+// Informer transform to trim object fields for memory efficiency.
+func informerObjectTrim(obj interface{}) (interface{}, error) {
+	if accessor, err := meta.Accessor(obj); err == nil {
+		accessor.SetManagedFields(nil)
+	}
+	if pod, ok := obj.(*kapi.Pod); ok {
+		pod.Spec.Volumes = []kapi.Volume{}
+		for i := range pod.Spec.Containers {
+			pod.Spec.Containers[i].Command = nil
+			pod.Spec.Containers[i].Args = nil
+			pod.Spec.Containers[i].Env = nil
+			pod.Spec.Containers[i].VolumeMounts = nil
+		}
+	}
+	return obj, nil
+}
+
+func newNetDefInformer(netdefClient netdefclient.Interface) (netdefinformer.SharedInformerFactory, cache.SharedIndexInformer) {
+	const resyncInterval time.Duration = 1 * time.Second
+
+	informerFactory := netdefinformer.NewSharedInformerFactoryWithOptions(netdefClient, resyncInterval)
+	netdefInformer := informerFactory.InformerFor(&netdefv1.NetworkAttachmentDefinition{}, func(client netdefclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return netdefinformerv1.NewNetworkAttachmentDefinitionInformer(
+			client,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	})
+
+	return informerFactory, netdefInformer
+}
+
+func newPodInformer(kubeClient kubernetes.Interface, nodeName string) (internalinterfaces.SharedInformerFactory, cache.SharedIndexInformer) {
+	var tweakFunc internalinterfaces.TweakListOptionsFunc
+	if nodeName != "" {
+		logging.Verbosef("Filtering pod watch for node %q", nodeName)
+		// Only watch for local pods
+		tweakFunc = func(opts *metav1.ListOptions) {
+			opts.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", nodeName).String()
+		}
+	}
+
+	const resyncInterval time.Duration = 1 * time.Second
+
+	informerFactory := informerfactory.NewSharedInformerFactoryWithOptions(kubeClient, resyncInterval, informerfactory.WithTransform(informerObjectTrim))
+	podInformer := informerFactory.InformerFor(&kapi.Pod{}, func(c kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return v1coreinformers.NewFilteredPodInformer(
+			c,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+			tweakFunc)
+	})
+
+	return informerFactory, podInformer
+}
+
+func isPerNodeCertEnabled(config *PerNodeCertificate) (bool, error) {
+	if config != nil && config.Enabled {
+		if config.BootstrapKubeconfig != "" && config.CertDir != "" {
+			return true, nil
+		}
+		return true, logging.Errorf("failed to configure PerNodeCertificate: enabled: %v, BootstrapKubeconfig: %q, CertDir: %q", config.Enabled, config.BootstrapKubeconfig, config.CertDir)
+	}
+	return false, nil
+}
+
 // NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
-func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte) (*Server, error) {
-	kubeClient, err := k8s.InClusterK8sClient()
-	if err != nil {
-		return nil, fmt.Errorf("error getting k8s client: %v", err)
+func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte, ignoreReadinessIndicator bool) (*Server, error) {
+	var kubeClient *k8s.ClientInfo
+	enabled, err := isPerNodeCertEnabled(daemonConfig.PerNodeCertificate)
+	if enabled {
+		if err != nil {
+			return nil, err
+		}
+		perNodeCertConfig := daemonConfig.PerNodeCertificate
+		nodeName := os.Getenv("MULTUS_NODE_NAME")
+		if nodeName == "" {
+			return nil, logging.Errorf("error getting node name for perNodeCertificate, please check manifest to have MULTUS_NODE_NAME")
+		}
+
+		certDuration := DefaultCertDuration
+		if perNodeCertConfig.CertDuration != "" {
+			certDuration, err = time.ParseDuration(perNodeCertConfig.CertDuration)
+			if err != nil {
+				return nil, logging.Errorf("failed to parse certDuration: %v", err)
+			}
+		}
+
+		kubeClient, err = k8s.PerNodeK8sClient(nodeName, perNodeCertConfig.BootstrapKubeconfig, certDuration, perNodeCertConfig.CertDir)
+		if err != nil {
+			return nil, logging.Errorf("error getting perNodeClient: %v", err)
+		}
+	} else {
+		kubeClient, err = k8s.InClusterK8sClient()
+		if err != nil {
+			return nil, fmt.Errorf("error getting k8s client: %v", err)
+		}
 	}
 
 	exec := invoke.Exec(nil)
@@ -140,21 +251,17 @@ func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte) (*Server
 			Stderr:    os.Stderr,
 			chrootDir: daemonConfig.ChrootDir,
 		}
-		types.ChrootMutex = &chrootExec.mu
 		exec = chrootExec
 		logging.Verbosef("server configured with chroot: %s", daemonConfig.ChrootDir)
 	}
 
-	return newCNIServer(daemonConfig.SocketDir, kubeClient, exec, serverConfig)
+	return newCNIServer(daemonConfig.SocketDir, kubeClient, exec, serverConfig, ignoreReadinessIndicator)
 }
 
-func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {
-
-	// preprocess server config to be used to override multus CNI config
-	// see extractCniData() for the detail
-	if servConfig != nil {
-		servConfig = bytes.Replace(servConfig, []byte("{"), []byte(","), 1)
-	}
+func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte, ignoreReadinessIndicator bool) (*Server, error) {
+	informerFactory, podInformer := newPodInformer(kubeClient.Client, os.Getenv("MULTUS_NODE_NAME"))
+	netdefInformerFactory, netdefInformer := newNetDefInformer(kubeClient.NetClient)
+	kubeClient.SetK8sClientInformers(podInformer, netdefInformer)
 
 	router := http.NewServeMux()
 	s := &Server{
@@ -174,7 +281,14 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 				[]string{"handler", "code", "method"},
 			),
 		},
+		informerFactory:          informerFactory,
+		podInformer:              podInformer,
+		netdefInformerFactory:    netdefInformerFactory,
+		netdefInformer:           netdefInformer,
+		ignoreReadinessIndicator: ignoreReadinessIndicator,
 	}
+	s.SetKeepAlivesEnabled(false)
+
 	// register metrics
 	prometheus.MustRegister(s.metrics.requestCounter)
 
@@ -225,7 +339,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 	// handle for '/healthz'
 	router.HandleFunc(api.MultusHealthAPIEndpoint, promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": api.MultusHealthAPIEndpoint}),
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if r.Method != http.MethodGet {
+			if r.Method != http.MethodGet && r.Method != http.MethodPost {
 				http.Error(w, fmt.Sprintf("Method not allowed"), http.StatusMethodNotAllowed)
 				return
 			}
@@ -244,6 +358,37 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 	return s, nil
 }
 
+// Start starts the server and begins serving on the given listener
+func (s *Server) Start(ctx context.Context, l net.Listener) {
+	s.informerFactory.Start(ctx.Done())
+	s.netdefInformerFactory.Start(ctx.Done())
+
+	// Give the initial sync some time to complete in large clusters, but
+	// don't wait forever
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), s.podInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	// Give the initial sync some time to complete in large clusters, but
+	// don't wait forever
+	waitCtx, waitCancel = context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), s.netdefInformer.HasSynced) {
+		logging.Errorf("failed to sync net-attach-def informer cache")
+	}
+	waitCancel()
+
+	go func() {
+		utilwait.UntilWithContext(ctx, func(_ context.Context) {
+			logging.Debugf("open for business")
+			if err := s.Serve(l); err != nil {
+				utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
+			}
+		}, 0)
+	}()
+}
+
 func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 	var cr api.Request
 	b, err := io.ReadAll(r.Body)
@@ -253,7 +398,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 	if err := json.Unmarshal(b, &cr); err != nil {
 		return nil, err
 	}
-	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	cmdType, cniCmdArgs, err := s.extractCniData(&cr, s.serverConfig)
 	if err != nil {
 		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
 	}
@@ -263,10 +408,10 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
 	}
 
-	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient)
+	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs)
 	if err != nil {
 		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+		return nil, fmt.Errorf("%s ERRORED: %v", printCmdArgs(cniCmdArgs), err)
 	}
 	return result, nil
 }
@@ -280,7 +425,7 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 	if err := json.Unmarshal(b, &cr); err != nil {
 		return nil, err
 	}
-	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	cmdType, cniCmdArgs, err := s.extractCniData(&cr, s.serverConfig)
 	if err != nil {
 		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
 	}
@@ -290,15 +435,50 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
 	}
 
-	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient, cr.InterfaceAttributes)
+	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, cr.InterfaceAttributes)
 	if err != nil {
 		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+		return nil, fmt.Errorf("%s ERRORED: %v", printCmdArgs(cniCmdArgs), err)
 	}
 	return result, nil
 }
 
-func extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
+func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, ignoreReadinessIndicator bool) ([]byte, error) {
+	if len(overrideConf) == 0 {
+		return cniConf, nil
+	}
+
+	var cni map[string]interface{}
+	if err := json.Unmarshal(cniConf, &cni); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall CNI config: %w", err)
+	}
+
+	var override map[string]interface{}
+	if err := json.Unmarshal(overrideConf, &override); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall CNI override config: %w", err)
+	}
+
+	// Copy each key of the override config into the CNI config except for
+	// a few specific keys
+	ignoreKeys := sets.NewString()
+	if ignoreReadinessIndicator {
+		ignoreKeys.Insert("readinessindicatorfile")
+	}
+	for overrideKey, overrideVal := range override {
+		if !ignoreKeys.Has(overrideKey) {
+			cni[overrideKey] = overrideVal
+		}
+	}
+
+	newBytes, err := json.Marshal(cni)
+	if err != nil {
+		return nil, fmt.Errorf("failed ot marshall new CNI config with overrides: %w", err)
+	}
+
+	return newBytes, nil
+}
+
+func (s *Server) extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
 	cmd, ok := cniRequest.Env["CNI_COMMAND"]
 	if !ok {
 		return "", nil, fmt.Errorf("unexpected or missing CNI_COMMAND")
@@ -325,18 +505,10 @@ func extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel
 	}
 	cniCmdArgs.Args = cniArgs
 
-	if overrideConf != nil {
-		// trim the close bracket from multus CNI config and put the server config
-		// to override CNI config with server config.
-		// note: if there are two or more value in same key, then the
-		// latest one is used at golang json implementation
-		idx := bytes.LastIndex(cniRequest.Config, []byte("}"))
-		if idx == -1 {
-			return "", nil, fmt.Errorf("invalid CNI config")
-		}
-		cniCmdArgs.StdinData = append(cniRequest.Config[:idx], overrideConf...)
-	} else {
-		cniCmdArgs.StdinData = cniRequest.Config
+	var err error
+	cniCmdArgs.StdinData, err = overrideCNIConfigWithServerConfig(cniRequest.Config, overrideConf, s.ignoreReadinessIndicator)
+	if err != nil {
+		return "", nil, err
 	}
 
 	return cmd, cniCmdArgs, nil
@@ -409,7 +581,7 @@ func podUID(kubeclient *k8s.ClientInfo, cniArgs map[string]string, podNamespace,
 	return uid, nil
 }
 
-func cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+func (s *Server) cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) ([]byte, error) {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -417,14 +589,14 @@ func cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kub
 	}
 
 	logging.Debugf("CmdAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	result, err := multus.CmdAdd(cmdArgs, exec, kubeClient)
+	result, err := multus.CmdAdd(cmdArgs, s.exec, s.kubeclient)
 	if err != nil {
 		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
 	}
 	return serializeResult(result)
 }
 
-func cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+func (s *Server) cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -432,10 +604,10 @@ func cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kub
 	}
 
 	logging.Debugf("CmdDel for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	return multus.CmdDel(cmdArgs, exec, kubeClient)
+	return multus.CmdDel(cmdArgs, s.exec, s.kubeclient)
 }
 
-func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+func (s *Server) cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -443,7 +615,29 @@ func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, k
 	}
 
 	logging.Debugf("CmdCheck for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	return multus.CmdCheck(cmdArgs, exec, kubeClient)
+	return multus.CmdCheck(cmdArgs, s.exec, s.kubeclient)
+}
+
+func (s *Server) cmdGC(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdGC for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdGC(cmdArgs, s.exec, s.kubeclient)
+}
+
+func (s *Server) cmdStatus(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdStatus for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdStatus(cmdArgs, s.exec, s.kubeclient)
 }
 
 func serializeResult(result cnitypes.Result) ([]byte, error) {
@@ -460,13 +654,13 @@ func serializeResult(result cnitypes.Result) ([]byte, error) {
 	return responseBytes, nil
 }
 
-func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+func (s *Server) cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
 		return nil, fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
 	}
-	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	pod, err := multus.GetPod(s.kubeclient, k8sArgs, false)
 	if err != nil {
 		return nil, err
 	}
@@ -493,7 +687,7 @@ func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 
 	logging.Debugf("CmdDelegateAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	result, err := multus.DelegateAdd(exec, kubeClient, pod, delegateCNIConf, rt, multusConfig)
+	result, err := multus.DelegateAdd(s.exec, s.kubeclient, pod, delegateCNIConf, rt, multusConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
 	}
@@ -501,26 +695,26 @@ func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 	return serializeResult(result)
 }
 
-func cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, _ *k8s.ClientInfo, multusConfig *types.NetConf) error {
+func (s *Server) cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
 	delegateCNIConf := &types.DelegateNetConf{}
 	if err := json.Unmarshal(cmdArgs.StdinData, delegateCNIConf); err != nil {
 		return err
 	}
 	delegateCNIConf.Bytes = cmdArgs.StdinData
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	return multus.DelegateCheck(exec, delegateCNIConf, rt, multusConfig)
+	return multus.DelegateCheck(s.exec, delegateCNIConf, rt, multusConfig)
 }
 
 // note: this function may send back error to the client. In cni spec, command DEL should NOT send any error
 // because container deletion follows cni DEL command. But in delegateDel case, container is not removed by
 // this delegateDel, hence we decide to send error message to the request sender.
-func cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf) error {
+func (s *Server) cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
 		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
 	}
-	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	pod, err := multus.GetPod(s.kubeclient, k8sArgs, false)
 	if err != nil {
 		return err
 	}
@@ -530,7 +724,7 @@ func cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 		return err
 	}
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	return multus.DelegateDel(exec, pod, delegateCNIConf, rt, multusConfig)
+	return multus.DelegateDel(s.exec, pod, delegateCNIConf, rt, multusConfig)
 }
 
 // LoadDaemonNetConf loads the configuration for the multus daemon

pkg/server/server_suite_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/server/server_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+// disable dot-imports only for testing
+//revive:disable:dot-imports
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Server", func() {
+	cniConf := []byte(`{
+	"binDir": "/var/lib/cni/bin",
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"cniVersion": "0.3.1",
+	"daemonSocketDir": "/run/multus/socket",
+	"globalNamespaces": "default,openshift-multus,openshift-sriov-network-operator",
+	"logLevel": "verbose",
+	"logToStderr": true,
+	"name": "multus-cni-network",
+	"namespaceIsolation": true,
+	"type": "multus-shim"
+}`)
+
+	serverConf := []byte(`{
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"readinessindicatorfile": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`)
+
+	Context("correctly overrides incoming CNI config with server config", func() {
+		newConf, err := overrideCNIConfigWithServerConfig(cniConf, serverConf, false)
+		Expect(err).ToNot(HaveOccurred())
+
+		// All server options except readinessindicatorfile should exist
+		// in the returned config
+		Expect(newConf).To(MatchJSON(`{
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"name": "multus-cni-network",
+	"type": "multus-shim",
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"readinessindicatorfile": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`))
+	})
+
+	Context("correctly overrides incoming CNI config with server config and ignores readinessindicatorfile", func() {
+		newConf, err := overrideCNIConfigWithServerConfig(cniConf, serverConf, true)
+		Expect(err).ToNot(HaveOccurred())
+
+		// All server options except readinessindicatorfile should exist
+		// in the returned config
+		Expect(newConf).To(MatchJSON(`{
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"name": "multus-cni-network",
+	"type": "multus-shim",
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`))
+	})
+})

pkg/server/thick_cni_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -30,8 +32,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
 
@@ -96,20 +96,27 @@ var _ = Describe(suiteName, func() {
 			containerID = "123456789"
 			ifaceName   = "eth0"
 			podName     = "my-little-pod"
+			configPath  = "/tmp/foo.multus.conf"
 		)
 
 		var (
 			cniServer *Server
 			K8sClient *k8s.ClientInfo
 			netns     ns.NetNS
+			ctx       context.Context
+			cancel    context.CancelFunc
 		)
 
 		BeforeEach(func() {
 			var err error
 			K8sClient = fakeK8sClient()
+			// Touch the default network file.
+			os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 
 			Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
-			cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, nil)
+
+			ctx, cancel = context.WithCancel(context.TODO())
+			cniServer, err = startCNIServer(ctx, thickPluginRunDir, K8sClient, nil)
 			Expect(err).NotTo(HaveOccurred())
 
 			netns, err = testutils.NewNS()
@@ -121,6 +128,12 @@ var _ = Describe(suiteName, func() {
 		})
 
 		AfterEach(func() {
+			cancel()
+			// Cleanup default network file.
+			if _, errStat := os.Stat(configPath); errStat == nil {
+				errRemove := os.Remove(configPath)
+				Expect(errRemove).NotTo(HaveOccurred())
+			}
 			unregisterMetrics(cniServer)
 			Expect(cniServer.Close()).To(Succeed())
 			Expect(teardownCNIEnv()).To(Succeed())
@@ -145,12 +158,15 @@ var _ = Describe(suiteName, func() {
 			containerID = "123456789"
 			ifaceName   = "eth0"
 			podName     = "my-little-pod"
+			configPath  = "/tmp/foo.multus.conf"
 		)
 
 		var (
 			cniServer *Server
 			K8sClient *k8s.ClientInfo
 			netns     ns.NetNS
+			ctx       context.Context
+			cancel    context.CancelFunc
 		)
 
 		BeforeEach(func() {
@@ -162,8 +178,12 @@ var _ = Describe(suiteName, func() {
 				"dummy_key2": "dummy_val2"
 			}`
 
+			// Touch the default network file.
+			os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 			Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
-			cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, []byte(dummyServerConfig))
+
+			ctx, cancel = context.WithCancel(context.TODO())
+			cniServer, err = startCNIServer(ctx, thickPluginRunDir, K8sClient, []byte(dummyServerConfig))
 			Expect(err).NotTo(HaveOccurred())
 
 			netns, err = testutils.NewNS()
@@ -175,6 +195,12 @@ var _ = Describe(suiteName, func() {
 		})
 
 		AfterEach(func() {
+			cancel()
+			// Cleanup default network file.
+			if _, errStat := os.Stat(configPath); errStat == nil {
+				errRemove := os.Remove(configPath)
+				Expect(errRemove).NotTo(HaveOccurred())
+			}
 			unregisterMetrics(cniServer)
 			Expect(cniServer.Close()).To(Succeed())
 			Expect(teardownCNIEnv()).To(Succeed())
@@ -200,7 +226,7 @@ func fakeK8sClient() *k8s.ClientInfo {
 	const magicNumber = 10
 	return &k8s.ClientInfo{
 		Client:        fake.NewSimpleClientset(),
-		NetClient:     netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient:     netfake.NewSimpleClientset(),
 		EventRecorder: record.NewFakeRecorder(magicNumber),
 	}
 }
@@ -245,10 +271,10 @@ func createFakePod(k8sClient *k8s.ClientInfo, podName string) error {
 	return err
 }
 
-func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) {
+func startCNIServer(ctx context.Context, runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) {
 	const period = 0
 
-	cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig)
+	cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig, true)
 	if err != nil {
 		return nil, err
 	}
@@ -258,12 +284,8 @@ func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte)
 		return nil, fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(runDir), err)
 	}
 
-	cniServer.SetKeepAlivesEnabled(false)
-	go utilwait.Forever(func() {
-		if err := cniServer.Serve(l); err != nil {
-			utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
-		}
-	}, period)
+	cniServer.Start(ctx, l)
+
 	return cniServer, nil
 }
 
@@ -280,7 +302,7 @@ func referenceConfig(thickPluginSocketDir string) string {
         "name": "node-cni-network",
         "type": "multus",
         "daemonSocketDir": "%s",
-        "defaultnetworkfile": "/tmp/foo.multus.conf",
+        "readinessindicatorfile": "/tmp/foo.multus.conf",
         "defaultnetworkwaitseconds": 3,
         "delegates": [{
             "name": "weave1",

pkg/server/types.go

@@ -16,12 +16,17 @@ package server
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/invoke"
 
 	"github.com/prometheus/client_golang/prometheus"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	"k8s.io/client-go/informers/internalinterfaces"
+	"k8s.io/client-go/tools/cache"
 )
 
 const (
@@ -31,6 +36,8 @@ const (
 	DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json"
 	// DefaultMultusRunDir specifies default RunDir for multus
 	DefaultMultusRunDir = "/run/multus/"
+	// DefaultCertDuration specifies default duration for certs in per-node-certs config
+	DefaultCertDuration = 10 * time.Minute
 )
 
 // Metrics represents server's metrics.
@@ -42,19 +49,34 @@ type Metrics struct {
 // the CNI shim requests issued when a pod is added / removed.
 type Server struct {
 	http.Server
-	rundir       string
-	kubeclient   *k8sclient.ClientInfo
-	exec         invoke.Exec
-	serverConfig []byte
-	metrics      *Metrics
+	rundir                string
+	kubeclient            *k8sclient.ClientInfo
+	exec                  invoke.Exec
+	serverConfig          []byte
+	metrics               *Metrics
+	informerFactory       internalinterfaces.SharedInformerFactory
+	podInformer           cache.SharedIndexInformer
+	netdefInformerFactory netdefinformer.SharedInformerFactory
+	netdefInformer        cache.SharedIndexInformer
+
+	ignoreReadinessIndicator bool
+}
+
+// PerNodeCertificate for auto certificate generation for per node
+type PerNodeCertificate struct {
+	Enabled             bool   `json:"enabled,omitempty"`
+	BootstrapKubeconfig string `json:"bootstrapKubeconfig,omitempty"`
+	CertDir             string `json:"certDir,omitempty"`
+	CertDuration        string `json:"certDuration,omitempty"`
 }
 
 // ControllerNetConf for the controller cni configuration
 type ControllerNetConf struct {
-	ChrootDir   string `json:"chrootDir,omitempty"`
-	LogFile     string `json:"logFile"`
-	LogLevel    string `json:"logLevel"`
-	LogToStderr bool   `json:"logToStderr,omitempty"`
+	ChrootDir          string              `json:"chrootDir,omitempty"`
+	LogFile            string              `json:"logFile"`
+	LogLevel           string              `json:"logLevel"`
+	LogToStderr        bool                `json:"logToStderr,omitempty"`
+	PerNodeCertificate *PerNodeCertificate `json:"perNodeCertificate,omitempty"`
 
 	MetricsPort *int `json:"metricsPort,omitempty"`
 

pkg/signals/signals.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2024 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package signals provides handling for os signals.
+package signals
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+
+// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned
+// which is canceled on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+func SetupSignalHandler() context.Context {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-c
+		cancel()
+		<-c
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx
+}

pkg/types/conf.go

@@ -20,15 +20,18 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"strings"
-	"sync"
+	"time"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -40,9 +43,6 @@ const (
 	defaultNonIsolatedNamespace   = "default"
 )
 
-// ChrootMutex provides lock to access host filesystem
-var ChrootMutex *sync.Mutex
-
 // LoadDelegateNetConfList reads DelegateNetConf from bytes
 func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error {
 	logging.Debugf("LoadDelegateNetConfList: %s, %v", string(bytes), delegateConf)
@@ -63,6 +63,112 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 	return nil
 }
 
+// ConvertNetworkConfigListToNetConfList converts a libcni.NetworkConfigList to a NetConfList
+func ConvertNetworkConfigListToNetConfList(ncList *libcni.NetworkConfigList) (*types.NetConfList, error) {
+	// Convert Plugins from []*libcni.PluginConfig to []*types.PluginConf
+	var plugins []*types.PluginConf
+	for _, plugin := range ncList.Plugins {
+		plugins = append(plugins, plugin.Network)
+	}
+
+	// Create NetConfList
+	netConfList := &types.NetConfList{
+		CNIVersion:   ncList.CNIVersion,
+		Name:         ncList.Name,
+		DisableCheck: ncList.DisableCheck,
+		DisableGC:    ncList.DisableGC,
+		Plugins:      plugins,
+	}
+
+	return netConfList, nil
+}
+
+// LoadDelegateNetConfFromConfList converts a libcni.NetworkConfigList into a DelegateNetConf structure
+func LoadDelegateNetConfFromConfList(confList *libcni.NetworkConfigList, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
+	var err error
+	logging.Debugf("LoadDelegateNetConfFromConfList: %v, %v, %s", confList, netElement, deviceID)
+
+	// Convert libcni.NetworkConfigList to NetConfList
+	netConfList, err := ConvertNetworkConfigListToNetConfList(confList)
+	if err != nil {
+		return nil, err
+	}
+
+	delegateConf := &DelegateNetConf{
+		Name:                 netConfList.Name,
+		ConfList:             *netConfList,
+		CNINetworkConfigList: *confList,
+		ConfListPlugin:       true,
+	}
+
+	// Convert the plugins back to bytes for consistency
+	pluginsBytes, err := json.Marshal(netConfList)
+	if err != nil {
+		return nil, logging.Errorf("LoadDelegateNetConfFromConfList: error marshaling netConfList: %v", err)
+	}
+	delegateConf.Bytes = pluginsBytes
+
+	if deviceID != "" {
+		pluginsBytes, err = addDeviceIDInConfList(pluginsBytes, deviceID)
+		if err != nil {
+			return nil, logging.Errorf("LoadDelegateNetConfFromConfList: failed to add deviceID in NetConfList bytes: %v", err)
+		}
+		delegateConf.ResourceName = resourceName
+		delegateConf.DeviceID = deviceID
+	}
+
+	if netElement != nil && netElement.CNIArgs != nil {
+		pluginsBytes, err = addCNIArgsInConfList(pluginsBytes, netElement.CNIArgs)
+		if err != nil {
+			return nil, logging.Errorf("LoadDelegateNetConfFromConfList: failed to add cni-args in NetConfList bytes: %v", err)
+		}
+		delegateConf.Bytes = pluginsBytes
+	}
+
+	if netElement != nil {
+		if netElement.Name != "" {
+			// Overwrite CNI config name with net-attach-def name
+			delegateConf.Name = fmt.Sprintf("%s/%s", netElement.Namespace, netElement.Name)
+		}
+		if netElement.InterfaceRequest != "" {
+			delegateConf.IfnameRequest = netElement.InterfaceRequest
+		}
+		if netElement.MacRequest != "" {
+			delegateConf.MacRequest = netElement.MacRequest
+		}
+		if netElement.IPRequest != nil {
+			delegateConf.IPRequest = netElement.IPRequest
+		}
+		if netElement.BandwidthRequest != nil {
+			delegateConf.BandwidthRequest = netElement.BandwidthRequest
+		}
+		if netElement.PortMappingsRequest != nil {
+			delegateConf.PortMappingsRequest = netElement.PortMappingsRequest
+		}
+		if netElement.GatewayRequest != nil {
+			var list []net.IP
+			if delegateConf.GatewayRequest != nil {
+				list = append(*delegateConf.GatewayRequest, *netElement.GatewayRequest...)
+			} else {
+				list = *netElement.GatewayRequest
+			}
+			delegateConf.GatewayRequest = &list
+		}
+		if netElement.InfinibandGUIDRequest != "" {
+			delegateConf.InfinibandGUIDRequest = netElement.InfinibandGUIDRequest
+		}
+		if netElement.DeviceID != "" {
+			if deviceID != "" {
+				logging.Debugf("Warning: Both RuntimeConfig and ResourceMap provide deviceID. Ignoring RuntimeConfig")
+			} else {
+				delegateConf.DeviceID = netElement.DeviceID
+			}
+		}
+	}
+
+	return delegateConf, nil
+}
+
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
 func LoadDelegateNetConf(bytes []byte, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
@@ -609,3 +715,37 @@ func CheckSystemNamespaces(namespace string, systemNamespaces []string) bool {
 	}
 	return false
 }
+
+// GetReadinessIndicatorFile waits for readinessIndicatorFile
+func GetReadinessIndicatorFile(readinessIndicatorFileRaw string) error {
+	cleanpath := filepath.Clean(readinessIndicatorFileRaw)
+	readinessIndicatorFile, err := filepath.Abs(cleanpath)
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path of readinessIndicatorFile: %v", err)
+	}
+
+	pollDuration := 1000 * time.Millisecond
+	pollTimeout := 45 * time.Second
+	return utilwait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
+		_, err := os.Stat(readinessIndicatorFile)
+		return err == nil, nil
+	})
+}
+
+// ReadinessIndicatorExistsNow reports if the readiness indicator exists immediately.
+func ReadinessIndicatorExistsNow(readinessIndicatorFileRaw string) (bool, error) {
+	cleanpath := filepath.Clean(readinessIndicatorFileRaw)
+	readinessIndicatorFile, err := filepath.Abs(cleanpath)
+	if err != nil {
+		return false, fmt.Errorf("failed to get absolute path of readinessIndicatorFile: %v", err)
+	}
+
+	_, err = os.Stat(readinessIndicatorFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}

pkg/types/conf_test.go

@@ -15,6 +15,8 @@
 
 package types
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"fmt"
@@ -599,7 +601,7 @@ var _ = Describe("config operations", func() {
 			StdinData: []byte(`{
     "name": "node-cni-network",
     "type": "multus",
-    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "readinessindicatorfile": "/tmp/foo.multus.conf",
     "defaultnetworkwaitseconds": 3,
     "delegates": [{
         "name": "weave1",
@@ -647,7 +649,7 @@ var _ = Describe("config operations", func() {
 			StdinData: []byte(`{
     "name": "node-cni-network",
     "type": "multus",
-    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "readinessindicatorfile": "/tmp/foo.multus.conf",
     "defaultnetworkwaitseconds": 3,
     "delegates": [{
         "name": "weave1",
@@ -702,9 +704,9 @@ var _ = Describe("config operations", func() {
 		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+		delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
-		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+		GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 
 		Expect(err).NotTo(HaveOccurred())
 	})
@@ -735,9 +737,9 @@ var _ = Describe("config operations", func() {
 		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 		fmt.Println("result.Version: ", result.Version())
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+		delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
-		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+		GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 
 		Expect(err).To(HaveOccurred())
 	})

pkg/types/types.go

@@ -18,10 +18,10 @@ package types
 import (
 	"net"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-
+	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -57,6 +57,7 @@ type NetConf struct {
 	NamespaceIsolation       bool     `json:"namespaceIsolation"`
 	RawNonIsolatedNamespaces string   `json:"globalNamespaces"`
 	NonIsolatedNamespaces    []string `json:"-"`
+	AuxiliaryCNIChainName    string   `json:"auxiliaryCNIChainName,omitempty"`
 
 	// Option to set system namespaces (to avoid to add defaultNetworks)
 	SystemNamespaces []string `json:"systemNamespaces"`
@@ -99,6 +100,7 @@ type BandwidthEntry struct {
 type DelegateNetConf struct {
 	Conf                  types.NetConf
 	ConfList              types.NetConfList
+	CNINetworkConfigList  libcni.NetworkConfigList
 	Name                  string
 	IfnameRequest         string          `json:"ifnameRequest,omitempty"`
 	MacRequest            string          `json:"macRequest,omitempty"`

vendor/github.com/cespare/xxhash/v2/README.md

@@ -3,8 +3,7 @@
 [![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2)
 [![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml)
 
-xxhash is a Go implementation of the 64-bit
-[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
+xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
 high-quality hashing algorithm that is much faster than anything in the Go
 standard library.
 
@@ -25,8 +24,11 @@ func (*Digest) WriteString(string) (int, error)
 func (*Digest) Sum64() uint64
 ```
 
-This implementation provides a fast pure-Go implementation and an even faster
-assembly implementation for amd64.
+The package is written with optimized pure Go and also contains even faster
+assembly implementations for amd64 and arm64. If desired, the `purego` build tag
+opts into using the Go code even on those architectures.
+
+[xxHash]: http://cyan4973.github.io/xxHash/
 
 ## Compatibility
 
@@ -45,19 +47,20 @@ I recommend using the latest release of Go.
 Here are some quick benchmarks comparing the pure-Go and assembly
 implementations of Sum64.
 
-| input size | purego | asm |
-| --- | --- | --- |
-| 5 B   |  979.66 MB/s |  1291.17 MB/s  |
-| 100 B | 7475.26 MB/s | 7973.40 MB/s  |
-| 4 KB  | 17573.46 MB/s | 17602.65 MB/s |
-| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
+| input size | purego    | asm       |
+| ---------- | --------- | --------- |
+| 4 B        |  1.3 GB/s |  1.2 GB/s |
+| 16 B       |  2.9 GB/s |  3.5 GB/s |
+| 100 B      |  6.9 GB/s |  8.1 GB/s |
+| 4 KB       | 11.7 GB/s | 16.7 GB/s |
+| 10 MB      | 12.0 GB/s | 17.3 GB/s |
 
-These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
-the following commands under Go 1.11.2:
+These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
+CPU using the following commands under Go 1.19.2:
 
 ```
-$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
-$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
+benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
+benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
 ```
 
 ## Projects using this package

vendor/github.com/cespare/xxhash/v2/testall.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eu -o pipefail
+
+# Small convenience script for running the tests with various combinations of
+# arch/tags. This assumes we're running on amd64 and have qemu available.
+
+go test ./...
+go test -tags purego ./...
+GOARCH=arm64 go test
+GOARCH=arm64 go test -tags purego

vendor/github.com/cespare/xxhash/v2/xxhash.go

@@ -16,19 +16,11 @@ const (
 	prime5 uint64 = 2870177450012600261
 )
 
-// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
-// possible in the Go code is worth a small (but measurable) performance boost
-// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
-// convenience in the Go code in a few places where we need to intentionally
-// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
-// result overflows a uint64).
-var (
-	prime1v = prime1
-	prime2v = prime2
-	prime3v = prime3
-	prime4v = prime4
-	prime5v = prime5
-)
+// Store the primes in an array as well.
+//
+// The consts are used when possible in Go code to avoid MOVs but we need a
+// contiguous array of the assembly code.
+var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
 
 // Digest implements hash.Hash64.
 type Digest struct {
@@ -50,10 +42,10 @@ func New() *Digest {
 
 // Reset clears the Digest's state so that it can be reused.
 func (d *Digest) Reset() {
-	d.v1 = prime1v + prime2
+	d.v1 = primes[0] + prime2
 	d.v2 = prime2
 	d.v3 = 0
-	d.v4 = -prime1v
+	d.v4 = -primes[0]
 	d.total = 0
 	d.n = 0
 }
@@ -69,21 +61,23 @@ func (d *Digest) Write(b []byte) (n int, err error) {
 	n = len(b)
 	d.total += uint64(n)
 
+	memleft := d.mem[d.n&(len(d.mem)-1):]
+
 	if d.n+n < 32 {
 		// This new data doesn't even fill the current block.
-		copy(d.mem[d.n:], b)
+		copy(memleft, b)
 		d.n += n
 		return
 	}
 
 	if d.n > 0 {
 		// Finish off the partial block.
-		copy(d.mem[d.n:], b)
+		c := copy(memleft, b)
 		d.v1 = round(d.v1, u64(d.mem[0:8]))
 		d.v2 = round(d.v2, u64(d.mem[8:16]))
 		d.v3 = round(d.v3, u64(d.mem[16:24]))
 		d.v4 = round(d.v4, u64(d.mem[24:32]))
-		b = b[32-d.n:]
+		b = b[c:]
 		d.n = 0
 	}
 
@@ -133,21 +127,20 @@ func (d *Digest) Sum64() uint64 {
 
 	h += d.total
 
-	i, end := 0, d.n
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(d.mem[i:i+8]))
+	b := d.mem[:d.n&(len(d.mem)-1)]
+	for ; len(b) >= 8; b = b[8:] {
+		k1 := round(0, u64(b[:8]))
 		h ^= k1
 		h = rol27(h)*prime1 + prime4
 	}
-	if i+4 <= end {
-		h ^= uint64(u32(d.mem[i:i+4])) * prime1
+	if len(b) >= 4 {
+		h ^= uint64(u32(b[:4])) * prime1
 		h = rol23(h)*prime2 + prime3
-		i += 4
+		b = b[4:]
 	}
-	for i < end {
-		h ^= uint64(d.mem[i]) * prime5
+	for ; len(b) > 0; b = b[1:] {
+		h ^= uint64(b[0]) * prime5
 		h = rol11(h) * prime1
-		i++
 	}
 
 	h ^= h >> 33

vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s

@@ -1,215 +1,209 @@
+//go:build !appengine && gc && !purego
 // +build !appengine
 // +build gc
 // +build !purego
 
 #include "textflag.h"
 
-// Register allocation:
-// AX	h
-// SI	pointer to advance through b
-// DX	n
-// BX	loop end
-// R8	v1, k1
-// R9	v2
-// R10	v3
-// R11	v4
-// R12	tmp
-// R13	prime1v
-// R14	prime2v
-// DI	prime4v
+// Registers:
+#define h      AX
+#define d      AX
+#define p      SI // pointer to advance through b
+#define n      DX
+#define end    BX // loop end
+#define v1     R8
+#define v2     R9
+#define v3     R10
+#define v4     R11
+#define x      R12
+#define prime1 R13
+#define prime2 R14
+#define prime4 DI
 
-// round reads from and advances the buffer pointer in SI.
-// It assumes that R13 has prime1v and R14 has prime2v.
-#define round(r) \
-	MOVQ  (SI), R12 \
-	ADDQ  $8, SI    \
-	IMULQ R14, R12  \
-	ADDQ  R12, r    \
-	ROLQ  $31, r    \
-	IMULQ R13, r
+#define round(acc, x) \
+	IMULQ prime2, x   \
+	ADDQ  x, acc      \
+	ROLQ  $31, acc    \
+	IMULQ prime1, acc
 
-// mergeRound applies a merge round on the two registers acc and val.
-// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
-#define mergeRound(acc, val) \
-	IMULQ R14, val \
-	ROLQ  $31, val \
-	IMULQ R13, val \
-	XORQ  val, acc \
-	IMULQ R13, acc \
-	ADDQ  DI, acc
+// round0 performs the operation x = round(0, x).
+#define round0(x) \
+	IMULQ prime2, x \
+	ROLQ  $31, x    \
+	IMULQ prime1, x
+
+// mergeRound applies a merge round on the two registers acc and x.
+// It assumes that prime1, prime2, and prime4 have been loaded.
+#define mergeRound(acc, x) \
+	round0(x)         \
+	XORQ  x, acc      \
+	IMULQ prime1, acc \
+	ADDQ  prime4, acc
+
+// blockLoop processes as many 32-byte blocks as possible,
+// updating v1, v2, v3, and v4. It assumes that there is at least one block
+// to process.
+#define blockLoop() \
+loop:  \
+	MOVQ +0(p), x  \
+	round(v1, x)   \
+	MOVQ +8(p), x  \
+	round(v2, x)   \
+	MOVQ +16(p), x \
+	round(v3, x)   \
+	MOVQ +24(p), x \
+	round(v4, x)   \
+	ADDQ $32, p    \
+	CMPQ p, end    \
+	JLE  loop
 
 // func Sum64(b []byte) uint64
-TEXT ·Sum64(SB), NOSPLIT, $0-32
+TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
 	// Load fixed primes.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
-	MOVQ ·prime4v(SB), DI
+	MOVQ ·primes+0(SB), prime1
+	MOVQ ·primes+8(SB), prime2
+	MOVQ ·primes+24(SB), prime4
 
 	// Load slice.
-	MOVQ b_base+0(FP), SI
-	MOVQ b_len+8(FP), DX
-	LEAQ (SI)(DX*1), BX
+	MOVQ b_base+0(FP), p
+	MOVQ b_len+8(FP), n
+	LEAQ (p)(n*1), end
 
 	// The first loop limit will be len(b)-32.
-	SUBQ $32, BX
+	SUBQ $32, end
 
 	// Check whether we have at least one block.
-	CMPQ DX, $32
+	CMPQ n, $32
 	JLT  noBlocks
 
 	// Set up initial state (v1, v2, v3, v4).
-	MOVQ R13, R8
-	ADDQ R14, R8
-	MOVQ R14, R9
-	XORQ R10, R10
-	XORQ R11, R11
-	SUBQ R13, R11
+	MOVQ prime1, v1
+	ADDQ prime2, v1
+	MOVQ prime2, v2
+	XORQ v3, v3
+	XORQ v4, v4
+	SUBQ prime1, v4
 
-	// Loop until SI > BX.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
+	blockLoop()
 
-	CMPQ SI, BX
-	JLE  blockLoop
+	MOVQ v1, h
+	ROLQ $1, h
+	MOVQ v2, x
+	ROLQ $7, x
+	ADDQ x, h
+	MOVQ v3, x
+	ROLQ $12, x
+	ADDQ x, h
+	MOVQ v4, x
+	ROLQ $18, x
+	ADDQ x, h
 
-	MOVQ R8, AX
-	ROLQ $1, AX
-	MOVQ R9, R12
-	ROLQ $7, R12
-	ADDQ R12, AX
-	MOVQ R10, R12
-	ROLQ $12, R12
-	ADDQ R12, AX
-	MOVQ R11, R12
-	ROLQ $18, R12
-	ADDQ R12, AX
-
-	mergeRound(AX, R8)
-	mergeRound(AX, R9)
-	mergeRound(AX, R10)
-	mergeRound(AX, R11)
+	mergeRound(h, v1)
+	mergeRound(h, v2)
+	mergeRound(h, v3)
+	mergeRound(h, v4)
 
 	JMP afterBlocks
 
 noBlocks:
-	MOVQ ·prime5v(SB), AX
+	MOVQ ·primes+32(SB), h
 
 afterBlocks:
-	ADDQ DX, AX
+	ADDQ n, h
 
-	// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
-	ADDQ $24, BX
+	ADDQ $24, end
+	CMPQ p, end
+	JG   try4
 
-	CMPQ SI, BX
-	JG   fourByte
+loop8:
+	MOVQ  (p), x
+	ADDQ  $8, p
+	round0(x)
+	XORQ  x, h
+	ROLQ  $27, h
+	IMULQ prime1, h
+	ADDQ  prime4, h
 
-wordLoop:
-	// Calculate k1.
-	MOVQ  (SI), R8
-	ADDQ  $8, SI
-	IMULQ R14, R8
-	ROLQ  $31, R8
-	IMULQ R13, R8
+	CMPQ p, end
+	JLE  loop8
 
-	XORQ  R8, AX
-	ROLQ  $27, AX
-	IMULQ R13, AX
-	ADDQ  DI, AX
+try4:
+	ADDQ $4, end
+	CMPQ p, end
+	JG   try1
 
-	CMPQ SI, BX
-	JLE  wordLoop
+	MOVL  (p), x
+	ADDQ  $4, p
+	IMULQ prime1, x
+	XORQ  x, h
 
-fourByte:
-	ADDQ $4, BX
-	CMPQ SI, BX
-	JG   singles
+	ROLQ  $23, h
+	IMULQ prime2, h
+	ADDQ  ·primes+16(SB), h
 
-	MOVL  (SI), R8
-	ADDQ  $4, SI
-	IMULQ R13, R8
-	XORQ  R8, AX
-
-	ROLQ  $23, AX
-	IMULQ R14, AX
-	ADDQ  ·prime3v(SB), AX
-
-singles:
-	ADDQ $4, BX
-	CMPQ SI, BX
+try1:
+	ADDQ $4, end
+	CMPQ p, end
 	JGE  finalize
 
-singlesLoop:
-	MOVBQZX (SI), R12
-	ADDQ    $1, SI
-	IMULQ   ·prime5v(SB), R12
-	XORQ    R12, AX
+loop1:
+	MOVBQZX (p), x
+	ADDQ    $1, p
+	IMULQ   ·primes+32(SB), x
+	XORQ    x, h
+	ROLQ    $11, h
+	IMULQ   prime1, h
 
-	ROLQ  $11, AX
-	IMULQ R13, AX
-
-	CMPQ SI, BX
-	JL   singlesLoop
+	CMPQ p, end
+	JL   loop1
 
 finalize:
-	MOVQ  AX, R12
-	SHRQ  $33, R12
-	XORQ  R12, AX
-	IMULQ R14, AX
-	MOVQ  AX, R12
-	SHRQ  $29, R12
-	XORQ  R12, AX
-	IMULQ ·prime3v(SB), AX
-	MOVQ  AX, R12
-	SHRQ  $32, R12
-	XORQ  R12, AX
+	MOVQ  h, x
+	SHRQ  $33, x
+	XORQ  x, h
+	IMULQ prime2, h
+	MOVQ  h, x
+	SHRQ  $29, x
+	XORQ  x, h
+	IMULQ ·primes+16(SB), h
+	MOVQ  h, x
+	SHRQ  $32, x
+	XORQ  x, h
 
-	MOVQ AX, ret+24(FP)
+	MOVQ h, ret+24(FP)
 	RET
 
-// writeBlocks uses the same registers as above except that it uses AX to store
-// the d pointer.
-
 // func writeBlocks(d *Digest, b []byte) int
-TEXT ·writeBlocks(SB), NOSPLIT, $0-40
+TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
 	// Load fixed primes needed for round.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
+	MOVQ ·primes+0(SB), prime1
+	MOVQ ·primes+8(SB), prime2
 
 	// Load slice.
-	MOVQ b_base+8(FP), SI
-	MOVQ b_len+16(FP), DX
-	LEAQ (SI)(DX*1), BX
-	SUBQ $32, BX
+	MOVQ b_base+8(FP), p
+	MOVQ b_len+16(FP), n
+	LEAQ (p)(n*1), end
+	SUBQ $32, end
 
 	// Load vN from d.
-	MOVQ d+0(FP), AX
-	MOVQ 0(AX), R8   // v1
-	MOVQ 8(AX), R9   // v2
-	MOVQ 16(AX), R10 // v3
-	MOVQ 24(AX), R11 // v4
+	MOVQ s+0(FP), d
+	MOVQ 0(d), v1
+	MOVQ 8(d), v2
+	MOVQ 16(d), v3
+	MOVQ 24(d), v4
 
 	// We don't need to check the loop condition here; this function is
 	// always called with at least one block of data to process.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
-
-	CMPQ SI, BX
-	JLE  blockLoop
+	blockLoop()
 
 	// Copy vN back to d.
-	MOVQ R8, 0(AX)
-	MOVQ R9, 8(AX)
-	MOVQ R10, 16(AX)
-	MOVQ R11, 24(AX)
+	MOVQ v1, 0(d)
+	MOVQ v2, 8(d)
+	MOVQ v3, 16(d)
+	MOVQ v4, 24(d)
 
-	// The number of bytes written is SI minus the old base pointer.
-	SUBQ b_base+8(FP), SI
-	MOVQ SI, ret+32(FP)
+	// The number of bytes written is p minus the old base pointer.
+	SUBQ b_base+8(FP), p
+	MOVQ p, ret+32(FP)
 
 	RET

vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s

@@ -0,0 +1,183 @@
+//go:build !appengine && gc && !purego
+// +build !appengine
+// +build gc
+// +build !purego
+
+#include "textflag.h"
+
+// Registers:
+#define digest	R1
+#define h	R2 // return value
+#define p	R3 // input pointer
+#define n	R4 // input length
+#define nblocks	R5 // n / 32
+#define prime1	R7
+#define prime2	R8
+#define prime3	R9
+#define prime4	R10
+#define prime5	R11
+#define v1	R12
+#define v2	R13
+#define v3	R14
+#define v4	R15
+#define x1	R20
+#define x2	R21
+#define x3	R22
+#define x4	R23
+
+#define round(acc, x) \
+	MADD prime2, acc, x, acc \
+	ROR  $64-31, acc         \
+	MUL  prime1, acc
+
+// round0 performs the operation x = round(0, x).
+#define round0(x) \
+	MUL prime2, x \
+	ROR $64-31, x \
+	MUL prime1, x
+
+#define mergeRound(acc, x) \
+	round0(x)                     \
+	EOR  x, acc                   \
+	MADD acc, prime4, prime1, acc
+
+// blockLoop processes as many 32-byte blocks as possible,
+// updating v1, v2, v3, and v4. It assumes that n >= 32.
+#define blockLoop() \
+	LSR     $5, n, nblocks  \
+	PCALIGN $16             \
+	loop:                   \
+	LDP.P   16(p), (x1, x2) \
+	LDP.P   16(p), (x3, x4) \
+	round(v1, x1)           \
+	round(v2, x2)           \
+	round(v3, x3)           \
+	round(v4, x4)           \
+	SUB     $1, nblocks     \
+	CBNZ    nblocks, loop
+
+// func Sum64(b []byte) uint64
+TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
+	LDP b_base+0(FP), (p, n)
+
+	LDP  ·primes+0(SB), (prime1, prime2)
+	LDP  ·primes+16(SB), (prime3, prime4)
+	MOVD ·primes+32(SB), prime5
+
+	CMP  $32, n
+	CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
+	BLT  afterLoop
+
+	ADD  prime1, prime2, v1
+	MOVD prime2, v2
+	MOVD $0, v3
+	NEG  prime1, v4
+
+	blockLoop()
+
+	ROR $64-1, v1, x1
+	ROR $64-7, v2, x2
+	ADD x1, x2
+	ROR $64-12, v3, x3
+	ROR $64-18, v4, x4
+	ADD x3, x4
+	ADD x2, x4, h
+
+	mergeRound(h, v1)
+	mergeRound(h, v2)
+	mergeRound(h, v3)
+	mergeRound(h, v4)
+
+afterLoop:
+	ADD n, h
+
+	TBZ   $4, n, try8
+	LDP.P 16(p), (x1, x2)
+
+	round0(x1)
+
+	// NOTE: here and below, sequencing the EOR after the ROR (using a
+	// rotated register) is worth a small but measurable speedup for small
+	// inputs.
+	ROR  $64-27, h
+	EOR  x1 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+	round0(x2)
+	ROR  $64-27, h
+	EOR  x2 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+try8:
+	TBZ    $3, n, try4
+	MOVD.P 8(p), x1
+
+	round0(x1)
+	ROR  $64-27, h
+	EOR  x1 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+try4:
+	TBZ     $2, n, try2
+	MOVWU.P 4(p), x2
+
+	MUL  prime1, x2
+	ROR  $64-23, h
+	EOR  x2 @> 64-23, h, h
+	MADD h, prime3, prime2, h
+
+try2:
+	TBZ     $1, n, try1
+	MOVHU.P 2(p), x3
+	AND     $255, x3, x1
+	LSR     $8, x3, x2
+
+	MUL prime5, x1
+	ROR $64-11, h
+	EOR x1 @> 64-11, h, h
+	MUL prime1, h
+
+	MUL prime5, x2
+	ROR $64-11, h
+	EOR x2 @> 64-11, h, h
+	MUL prime1, h
+
+try1:
+	TBZ   $0, n, finalize
+	MOVBU (p), x4
+
+	MUL prime5, x4
+	ROR $64-11, h
+	EOR x4 @> 64-11, h, h
+	MUL prime1, h
+
+finalize:
+	EOR h >> 33, h
+	MUL prime2, h
+	EOR h >> 29, h
+	MUL prime3, h
+	EOR h >> 32, h
+
+	MOVD h, ret+24(FP)
+	RET
+
+// func writeBlocks(d *Digest, b []byte) int
+TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
+	LDP ·primes+0(SB), (prime1, prime2)
+
+	// Load state. Assume v[1-4] are stored contiguously.
+	MOVD d+0(FP), digest
+	LDP  0(digest), (v1, v2)
+	LDP  16(digest), (v3, v4)
+
+	LDP b_base+8(FP), (p, n)
+
+	blockLoop()
+
+	// Store updated state.
+	STP (v1, v2), 0(digest)
+	STP (v3, v4), 16(digest)
+
+	BIC  $31, n
+	MOVD n, ret+32(FP)
+	RET

vendor/github.com/cespare/xxhash/v2/xxhash_asm.go

@@ -1,3 +1,5 @@
+//go:build (amd64 || arm64) && !appengine && gc && !purego
+// +build amd64 arm64
 // +build !appengine
 // +build gc
 // +build !purego

vendor/github.com/cespare/xxhash/v2/xxhash_other.go

@@ -1,4 +1,5 @@
-// +build !amd64 appengine !gc purego
+//go:build (!amd64 && !arm64) || appengine || !gc || purego
+// +build !amd64,!arm64 appengine !gc purego
 
 package xxhash
 
@@ -14,10 +15,10 @@ func Sum64(b []byte) uint64 {
 	var h uint64
 
 	if n >= 32 {
-		v1 := prime1v + prime2
+		v1 := primes[0] + prime2
 		v2 := prime2
 		v3 := uint64(0)
-		v4 := -prime1v
+		v4 := -primes[0]
 		for len(b) >= 32 {
 			v1 = round(v1, u64(b[0:8:len(b)]))
 			v2 = round(v2, u64(b[8:16:len(b)]))
@@ -36,19 +37,18 @@ func Sum64(b []byte) uint64 {
 
 	h += uint64(n)
 
-	i, end := 0, len(b)
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(b[i:i+8:len(b)]))
+	for ; len(b) >= 8; b = b[8:] {
+		k1 := round(0, u64(b[:8]))
 		h ^= k1
 		h = rol27(h)*prime1 + prime4
 	}
-	if i+4 <= end {
-		h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
+	if len(b) >= 4 {
+		h ^= uint64(u32(b[:4])) * prime1
 		h = rol23(h)*prime2 + prime3
-		i += 4
+		b = b[4:]
 	}
-	for ; i < end; i++ {
-		h ^= uint64(b[i]) * prime5
+	for ; len(b) > 0; b = b[1:] {
+		h ^= uint64(b[0]) * prime5
 		h = rol11(h) * prime1
 	}
 

vendor/github.com/cespare/xxhash/v2/xxhash_safe.go

@@ -1,3 +1,4 @@
+//go:build appengine
 // +build appengine
 
 // This file contains the safe implementations of otherwise unsafe-using code.

vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go

@@ -1,3 +1,4 @@
+//go:build !appengine
 // +build !appengine
 
 // This file encapsulates usage of unsafe.
@@ -11,7 +12,7 @@ import (
 
 // In the future it's possible that compiler optimizations will make these
 // XxxString functions unnecessary by realizing that calls such as
-// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205.
+// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205.
 // If that happens, even if we keep these functions they can be replaced with
 // the trivial safe code.
 

vendor/github.com/containernetworking/cni/libcni/api.go

@@ -15,7 +15,7 @@
 package libcni
 
 // Note this is the actual implementation of the CNI specification, which
-// is reflected in the https://github.com/containernetworking/cni/blob/master/SPEC.md file
+// is reflected in the SPEC.md file.
 // it is typically bundled into runtime providers (i.e. containerd or cri-o would use this
 // before calling runc or hcsshim).  It is also bundled into CNI providers as well, for example,
 // to add an IP to a container, to parse the configuration of the CNI and so on.
@@ -23,10 +23,11 @@ package libcni
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/containernetworking/cni/pkg/invoke"
@@ -38,6 +39,8 @@ import (
 
 var (
 	CacheDir = "/var/lib/cni"
+	// slightly awkward wording to preserve anyone matching on error strings
+	ErrorCheckNotSupp = fmt.Errorf("does not support the CHECK command")
 )
 
 const (
@@ -64,17 +67,37 @@ type RuntimeConf struct {
 	CacheDir string
 }
 
-type NetworkConfig struct {
-	Network *types.NetConf
+// Use PluginConfig instead of NetworkConfig, the NetworkConfig
+// backwards-compat alias will be removed in a future release.
+type NetworkConfig = PluginConfig
+
+type PluginConfig struct {
+	Network *types.PluginConf
 	Bytes   []byte
 }
 
 type NetworkConfigList struct {
-	Name         string
-	CNIVersion   string
-	DisableCheck bool
-	Plugins      []*NetworkConfig
-	Bytes        []byte
+	Name                   string
+	CNIVersion             string
+	DisableCheck           bool
+	DisableGC              bool
+	LoadOnlyInlinedPlugins bool
+	Plugins                []*PluginConfig
+	Bytes                  []byte
+}
+
+type NetworkAttachment struct {
+	ContainerID    string
+	Network        string
+	IfName         string
+	Config         []byte
+	NetNS          string
+	CniArgs        [][2]string
+	CapabilityArgs map[string]interface{}
+}
+
+type GCArgs struct {
+	ValidAttachments []types.GCAttachment
 }
 
 type CNI interface {
@@ -84,14 +107,21 @@ type CNI interface {
 	GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
 	GetNetworkListCachedConfig(net *NetworkConfigList, rt *RuntimeConf) ([]byte, *RuntimeConf, error)
 
-	AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
-	CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
-	DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
-	GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
-	GetNetworkCachedConfig(net *NetworkConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error)
+	AddNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) (types.Result, error)
+	CheckNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) error
+	DelNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) error
+	GetNetworkCachedResult(net *PluginConfig, rt *RuntimeConf) (types.Result, error)
+	GetNetworkCachedConfig(net *PluginConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error)
 
 	ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error)
-	ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error)
+	ValidateNetwork(ctx context.Context, net *PluginConfig) ([]string, error)
+
+	GCNetworkList(ctx context.Context, net *NetworkConfigList, args *GCArgs) error
+	GetStatusNetworkList(ctx context.Context, net *NetworkConfigList) error
+
+	GetCachedAttachments(containerID string) ([]*NetworkAttachment, error)
+
+	GetVersionInfo(ctx context.Context, pluginType string) (version.PluginInfo, error)
 }
 
 type CNIConfig struct {
@@ -122,7 +152,7 @@ func NewCNIConfigWithCacheDir(path []string, cacheDir string, exec invoke.Exec)
 	}
 }
 
-func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) {
+func buildOneConfig(name, cniVersion string, orig *PluginConfig, prevResult types.Result, rt *RuntimeConf) (*PluginConfig, error) {
 	var err error
 
 	inject := map[string]interface{}{
@@ -139,8 +169,11 @@ func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult typ
 	if err != nil {
 		return nil, err
 	}
+	if rt != nil {
+		return injectRuntimeConfig(orig, rt)
+	}
 
-	return injectRuntimeConfig(orig, rt)
+	return orig, nil
 }
 
 // This function takes a libcni RuntimeConf structure and injects values into
@@ -155,7 +188,7 @@ func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult typ
 // capabilities include "portMappings", and the CapabilityArgs map includes a
 // "portMappings" key, that key and its value are added to the "runtimeConfig"
 // dictionary to be passed to the plugin's stdin.
-func injectRuntimeConfig(orig *NetworkConfig, rt *RuntimeConf) (*NetworkConfig, error) {
+func injectRuntimeConfig(orig *PluginConfig, rt *RuntimeConf) (*PluginConfig, error) {
 	var err error
 
 	rc := make(map[string]interface{})
@@ -195,6 +228,7 @@ type cachedInfo struct {
 	Config         []byte                 `json:"config"`
 	IfName         string                 `json:"ifName"`
 	NetworkName    string                 `json:"networkName"`
+	NetNS          string                 `json:"netns,omitempty"`
 	CniArgs        [][2]string            `json:"cniArgs,omitempty"`
 	CapabilityArgs map[string]interface{} `json:"capabilityArgs,omitempty"`
 	RawResult      map[string]interface{} `json:"result,omitempty"`
@@ -229,6 +263,7 @@ func (c *CNIConfig) cacheAdd(result types.Result, config []byte, netName string,
 		Config:         config,
 		IfName:         rt.IfName,
 		NetworkName:    netName,
+		NetNS:          rt.NetNS,
 		CniArgs:        rt.Args,
 		CapabilityArgs: rt.CapabilityArgs,
 	}
@@ -254,11 +289,11 @@ func (c *CNIConfig) cacheAdd(result types.Result, config []byte, netName string,
 	if err != nil {
 		return err
 	}
-	if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil {
+	if err := os.MkdirAll(filepath.Dir(fname), 0o700); err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(fname, newBytes, 0600)
+	return os.WriteFile(fname, newBytes, 0o600)
 }
 
 func (c *CNIConfig) cacheDel(netName string, rt *RuntimeConf) error {
@@ -277,7 +312,7 @@ func (c *CNIConfig) getCachedConfig(netName string, rt *RuntimeConf) ([]byte, *R
 	if err != nil {
 		return nil, nil, err
 	}
-	bytes, err = ioutil.ReadFile(fname)
+	bytes, err = os.ReadFile(fname)
 	if err != nil {
 		// Ignore read errors; the cached result may not exist on-disk
 		return nil, nil, nil
@@ -305,7 +340,7 @@ func (c *CNIConfig) getLegacyCachedResult(netName, cniVersion string, rt *Runtim
 	if err != nil {
 		return nil, err
 	}
-	data, err := ioutil.ReadFile(fname)
+	data, err := os.ReadFile(fname)
 	if err != nil {
 		// Ignore read errors; the cached result may not exist on-disk
 		return nil, nil
@@ -333,7 +368,7 @@ func (c *CNIConfig) getCachedResult(netName, cniVersion string, rt *RuntimeConf)
 	if err != nil {
 		return nil, err
 	}
-	fdata, err := ioutil.ReadFile(fname)
+	fdata, err := os.ReadFile(fname)
 	if err != nil {
 		// Ignore read errors; the cached result may not exist on-disk
 		return nil, nil
@@ -374,7 +409,7 @@ func (c *CNIConfig) GetNetworkListCachedResult(list *NetworkConfigList, rt *Runt
 
 // GetNetworkCachedResult returns the cached Result of the previous
 // AddNetwork() operation for a network, or an error.
-func (c *CNIConfig) GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) {
+func (c *CNIConfig) GetNetworkCachedResult(net *PluginConfig, rt *RuntimeConf) (types.Result, error) {
 	return c.getCachedResult(net.Network.Name, net.Network.CNIVersion, rt)
 }
 
@@ -386,11 +421,73 @@ func (c *CNIConfig) GetNetworkListCachedConfig(list *NetworkConfigList, rt *Runt
 
 // GetNetworkCachedConfig copies the input RuntimeConf to output
 // RuntimeConf with fields updated with info from the cached Config.
-func (c *CNIConfig) GetNetworkCachedConfig(net *NetworkConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error) {
+func (c *CNIConfig) GetNetworkCachedConfig(net *PluginConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error) {
 	return c.getCachedConfig(net.Network.Name, rt)
 }
 
-func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
+// GetCachedAttachments returns a list of network attachments from the cache.
+// The returned list will be filtered by the containerID if the value is not empty.
+func (c *CNIConfig) GetCachedAttachments(containerID string) ([]*NetworkAttachment, error) {
+	dirPath := filepath.Join(c.getCacheDir(&RuntimeConf{}), "results")
+	entries, err := os.ReadDir(dirPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	fileNames := make([]string, 0, len(entries))
+	for _, e := range entries {
+		fileNames = append(fileNames, e.Name())
+	}
+	sort.Strings(fileNames)
+
+	attachments := []*NetworkAttachment{}
+	for _, fname := range fileNames {
+		if len(containerID) > 0 {
+			part := fmt.Sprintf("-%s-", containerID)
+			pos := strings.Index(fname, part)
+			if pos <= 0 || pos+len(part) >= len(fname) {
+				continue
+			}
+		}
+
+		cacheFile := filepath.Join(dirPath, fname)
+		bytes, err := os.ReadFile(cacheFile)
+		if err != nil {
+			continue
+		}
+
+		cachedInfo := cachedInfo{}
+
+		if err := json.Unmarshal(bytes, &cachedInfo); err != nil {
+			continue
+		}
+		if cachedInfo.Kind != CNICacheV1 {
+			continue
+		}
+		if len(containerID) > 0 && cachedInfo.ContainerID != containerID {
+			continue
+		}
+		if cachedInfo.IfName == "" || cachedInfo.NetworkName == "" {
+			continue
+		}
+
+		attachments = append(attachments, &NetworkAttachment{
+			ContainerID:    cachedInfo.ContainerID,
+			Network:        cachedInfo.NetworkName,
+			IfName:         cachedInfo.IfName,
+			Config:         cachedInfo.Config,
+			NetNS:          cachedInfo.NetNS,
+			CniArgs:        cachedInfo.CniArgs,
+			CapabilityArgs: cachedInfo.CapabilityArgs,
+		})
+	}
+	return attachments, nil
+}
+
+func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *PluginConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
 	c.ensureExec()
 	pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
 	if err != nil {
@@ -432,7 +529,7 @@ func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList,
 	return result, nil
 }
 
-func (c *CNIConfig) checkNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error {
+func (c *CNIConfig) checkNetwork(ctx context.Context, name, cniVersion string, net *PluginConfig, prevResult types.Result, rt *RuntimeConf) error {
 	c.ensureExec()
 	pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
 	if err != nil {
@@ -453,7 +550,7 @@ func (c *CNIConfig) CheckNetworkList(ctx context.Context, list *NetworkConfigLis
 	if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil {
 		return err
 	} else if !gtet {
-		return fmt.Errorf("configuration version %q does not support the CHECK command", list.CNIVersion)
+		return fmt.Errorf("configuration version %q %w", list.CNIVersion, ErrorCheckNotSupp)
 	}
 
 	if list.DisableCheck {
@@ -474,7 +571,7 @@ func (c *CNIConfig) CheckNetworkList(ctx context.Context, list *NetworkConfigLis
 	return nil
 }
 
-func (c *CNIConfig) delNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error {
+func (c *CNIConfig) delNetwork(ctx context.Context, name, cniVersion string, net *PluginConfig, prevResult types.Result, rt *RuntimeConf) error {
 	c.ensureExec()
 	pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
 	if err != nil {
@@ -497,9 +594,9 @@ func (c *CNIConfig) DelNetworkList(ctx context.Context, list *NetworkConfigList,
 	if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil {
 		return err
 	} else if gtet {
-		cachedResult, err = c.getCachedResult(list.Name, list.CNIVersion, rt)
-		if err != nil {
-			return fmt.Errorf("failed to get network %q cached result: %w", list.Name, err)
+		if cachedResult, err = c.getCachedResult(list.Name, list.CNIVersion, rt); err != nil {
+			_ = c.cacheDel(list.Name, rt)
+			cachedResult = nil
 		}
 	}
 
@@ -509,12 +606,13 @@ func (c *CNIConfig) DelNetworkList(ctx context.Context, list *NetworkConfigList,
 			return fmt.Errorf("plugin %s failed (delete): %w", pluginDescription(net.Network), err)
 		}
 	}
+
 	_ = c.cacheDel(list.Name, rt)
 
 	return nil
 }
 
-func pluginDescription(net *types.NetConf) string {
+func pluginDescription(net *types.PluginConf) string {
 	if net == nil {
 		return "<missing>"
 	}
@@ -528,7 +626,7 @@ func pluginDescription(net *types.NetConf) string {
 }
 
 // AddNetwork executes the plugin with the ADD command
-func (c *CNIConfig) AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error) {
+func (c *CNIConfig) AddNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) (types.Result, error) {
 	result, err := c.addNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, nil, rt)
 	if err != nil {
 		return nil, err
@@ -542,12 +640,12 @@ func (c *CNIConfig) AddNetwork(ctx context.Context, net *NetworkConfig, rt *Runt
 }
 
 // CheckNetwork executes the plugin with the CHECK command
-func (c *CNIConfig) CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error {
+func (c *CNIConfig) CheckNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) error {
 	// CHECK was added in CNI spec version 0.4.0 and higher
 	if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil {
 		return err
 	} else if !gtet {
-		return fmt.Errorf("configuration version %q does not support the CHECK command", net.Network.CNIVersion)
+		return fmt.Errorf("configuration version %q %w", net.Network.CNIVersion, ErrorCheckNotSupp)
 	}
 
 	cachedResult, err := c.getCachedResult(net.Network.Name, net.Network.CNIVersion, rt)
@@ -558,7 +656,7 @@ func (c *CNIConfig) CheckNetwork(ctx context.Context, net *NetworkConfig, rt *Ru
 }
 
 // DelNetwork executes the plugin with the DEL command
-func (c *CNIConfig) DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error {
+func (c *CNIConfig) DelNetwork(ctx context.Context, net *PluginConfig, rt *RuntimeConf) error {
 	var cachedResult types.Result
 
 	// Cached result on DEL was added in CNI spec version 0.4.0 and higher
@@ -618,7 +716,7 @@ func (c *CNIConfig) ValidateNetworkList(ctx context.Context, list *NetworkConfig
 // ValidateNetwork checks that a configuration is reasonably valid.
 // It uses the same logic as ValidateNetworkList)
 // Returns a list of capabilities
-func (c *CNIConfig) ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error) {
+func (c *CNIConfig) ValidateNetwork(ctx context.Context, net *PluginConfig) ([]string, error) {
 	caps := []string{}
 	for c, ok := range net.Network.Capabilities {
 		if ok {
@@ -666,6 +764,129 @@ func (c *CNIConfig) GetVersionInfo(ctx context.Context, pluginType string) (vers
 	return invoke.GetVersionInfo(ctx, pluginPath, c.exec)
 }
 
+// GCNetworkList will do two things
+// - dump the list of cached attachments, and issue deletes as necessary
+// - issue a GC to the underlying plugins (if the version is high enough)
+func (c *CNIConfig) GCNetworkList(ctx context.Context, list *NetworkConfigList, args *GCArgs) error {
+	// If DisableGC is set, then don't bother GCing at all.
+	if list.DisableGC {
+		return nil
+	}
+
+	// First, get the list of cached attachments
+	cachedAttachments, err := c.GetCachedAttachments("")
+	if err != nil {
+		return nil
+	}
+
+	var validAttachments map[types.GCAttachment]interface{}
+	if args != nil {
+		validAttachments = make(map[types.GCAttachment]interface{}, len(args.ValidAttachments))
+		for _, a := range args.ValidAttachments {
+			validAttachments[a] = nil
+		}
+	}
+
+	var errs []error
+
+	for _, cachedAttachment := range cachedAttachments {
+		if cachedAttachment.Network != list.Name {
+			continue
+		}
+		// we found this attachment
+		gca := types.GCAttachment{
+			ContainerID: cachedAttachment.ContainerID,
+			IfName:      cachedAttachment.IfName,
+		}
+		if _, ok := validAttachments[gca]; ok {
+			continue
+		}
+		// otherwise, this attachment wasn't valid and we should issue a CNI DEL
+		rt := RuntimeConf{
+			ContainerID:    cachedAttachment.ContainerID,
+			NetNS:          cachedAttachment.NetNS,
+			IfName:         cachedAttachment.IfName,
+			Args:           cachedAttachment.CniArgs,
+			CapabilityArgs: cachedAttachment.CapabilityArgs,
+		}
+		if err := c.DelNetworkList(ctx, list, &rt); err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete stale attachment %s %s: %w", rt.ContainerID, rt.IfName, err))
+		}
+	}
+
+	// now, if the version supports it, issue a GC
+	if gt, _ := version.GreaterThanOrEqualTo(list.CNIVersion, "1.1.0"); gt {
+		inject := map[string]interface{}{
+			"name":       list.Name,
+			"cniVersion": list.CNIVersion,
+		}
+		if args != nil {
+			inject["cni.dev/valid-attachments"] = args.ValidAttachments
+			// #1101: spec used incorrect variable name
+			inject["cni.dev/attachments"] = args.ValidAttachments
+		}
+
+		for _, plugin := range list.Plugins {
+			// build config here
+			pluginConfig, err := InjectConf(plugin, inject)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to generate configuration to GC plugin %s: %w", plugin.Network.Type, err))
+			}
+			if err := c.gcNetwork(ctx, pluginConfig); err != nil {
+				errs = append(errs, fmt.Errorf("failed to GC plugin %s: %w", plugin.Network.Type, err))
+			}
+		}
+	}
+
+	return errors.Join(errs...)
+}
+
+func (c *CNIConfig) gcNetwork(ctx context.Context, net *PluginConfig) error {
+	c.ensureExec()
+	pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
+	if err != nil {
+		return err
+	}
+	args := c.args("GC", &RuntimeConf{})
+
+	return invoke.ExecPluginWithoutResult(ctx, pluginPath, net.Bytes, args, c.exec)
+}
+
+func (c *CNIConfig) GetStatusNetworkList(ctx context.Context, list *NetworkConfigList) error {
+	// If the version doesn't support status, abort.
+	if gt, _ := version.GreaterThanOrEqualTo(list.CNIVersion, "1.1.0"); !gt {
+		return nil
+	}
+
+	inject := map[string]interface{}{
+		"name":       list.Name,
+		"cniVersion": list.CNIVersion,
+	}
+
+	for _, plugin := range list.Plugins {
+		// build config here
+		pluginConfig, err := InjectConf(plugin, inject)
+		if err != nil {
+			return fmt.Errorf("failed to generate configuration to get plugin STATUS %s: %w", plugin.Network.Type, err)
+		}
+		if err := c.getStatusNetwork(ctx, pluginConfig); err != nil {
+			return err // Don't collect errors here, so we return a clean error code.
+		}
+	}
+	return nil
+}
+
+func (c *CNIConfig) getStatusNetwork(ctx context.Context, net *PluginConfig) error {
+	c.ensureExec()
+	pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
+	if err != nil {
+		return err
+	}
+	args := c.args("STATUS", &RuntimeConf{})
+
+	return invoke.ExecPluginWithoutResult(ctx, pluginPath, net.Bytes, args, c.exec)
+}
+
 // =====
 func (c *CNIConfig) args(action string, rt *RuntimeConf) *invoke.Args {
 	return &invoke.Args{

vendor/github.com/containernetworking/cni/libcni/conf.go

@@ -16,13 +16,16 @@ package libcni
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
+	"strings"
 
 	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/version"
 )
 
 type NotFoundError struct {
@@ -42,9 +45,16 @@ func (e NoConfigsFoundError) Error() string {
 	return fmt.Sprintf(`no net configurations found in %s`, e.Dir)
 }
 
-func ConfFromBytes(bytes []byte) (*NetworkConfig, error) {
-	conf := &NetworkConfig{Bytes: bytes, Network: &types.NetConf{}}
-	if err := json.Unmarshal(bytes, conf.Network); err != nil {
+// This will not validate that the plugins actually belong to the netconfig by ensuring
+// that they are loaded from a directory named after the networkName, relative to the network config.
+//
+// Since here we are just accepting raw bytes, the caller is responsible for ensuring that the plugin
+// config provided here actually "belongs" to the networkconfig in question.
+func NetworkPluginConfFromBytes(pluginConfBytes []byte) (*PluginConfig, error) {
+	// TODO why are we creating a struct that holds both the byte representation and the deserialized
+	// representation, and returning that, instead of just returning the deserialized representation?
+	conf := &PluginConfig{Bytes: pluginConfBytes, Network: &types.PluginConf{}}
+	if err := json.Unmarshal(pluginConfBytes, conf.Network); err != nil {
 		return nil, fmt.Errorf("error parsing configuration: %w", err)
 	}
 	if conf.Network.Type == "" {
@@ -53,17 +63,35 @@ func ConfFromBytes(bytes []byte) (*NetworkConfig, error) {
 	return conf, nil
 }
 
-func ConfFromFile(filename string) (*NetworkConfig, error) {
-	bytes, err := ioutil.ReadFile(filename)
+// Given a path to a directory containing a network configuration, and the name of a network,
+// loads all plugin definitions found at path `networkConfPath/networkName/*.conf`
+func NetworkPluginConfsFromFiles(networkConfPath, networkName string) ([]*PluginConfig, error) {
+	var pConfs []*PluginConfig
+
+	pluginConfPath := filepath.Join(networkConfPath, networkName)
+
+	pluginConfFiles, err := ConfFiles(pluginConfPath, []string{".conf"})
 	if err != nil {
-		return nil, fmt.Errorf("error reading %s: %w", filename, err)
+		return nil, fmt.Errorf("failed to read plugin config files in %s: %w", pluginConfPath, err)
 	}
-	return ConfFromBytes(bytes)
+
+	for _, pluginConfFile := range pluginConfFiles {
+		pluginConfBytes, err := os.ReadFile(pluginConfFile)
+		if err != nil {
+			return nil, fmt.Errorf("error reading %s: %w", pluginConfFile, err)
+		}
+		pluginConf, err := NetworkPluginConfFromBytes(pluginConfBytes)
+		if err != nil {
+			return nil, err
+		}
+		pConfs = append(pConfs, pluginConf)
+	}
+	return pConfs, nil
 }
 
-func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) {
+func NetworkConfFromBytes(confBytes []byte) (*NetworkConfigList, error) {
 	rawList := make(map[string]interface{})
-	if err := json.Unmarshal(bytes, &rawList); err != nil {
+	if err := json.Unmarshal(confBytes, &rawList); err != nil {
 		return nil, fmt.Errorf("error parsing configuration list: %w", err)
 	}
 
@@ -85,26 +113,115 @@ func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) {
 		}
 	}
 
-	disableCheck := false
-	if rawDisableCheck, ok := rawList["disableCheck"]; ok {
-		disableCheck, ok = rawDisableCheck.(bool)
+	rawVersions, ok := rawList["cniVersions"]
+	if ok {
+		// Parse the current package CNI version
+		rvs, ok := rawVersions.([]interface{})
 		if !ok {
-			return nil, fmt.Errorf("error parsing configuration list: invalid disableCheck type %T", rawDisableCheck)
+			return nil, fmt.Errorf("error parsing configuration list: invalid type for cniVersions: %T", rvs)
+		}
+		vs := make([]string, 0, len(rvs))
+		for i, rv := range rvs {
+			v, ok := rv.(string)
+			if !ok {
+				return nil, fmt.Errorf("error parsing configuration list: invalid type for cniVersions index %d: %T", i, rv)
+			}
+			gt, err := version.GreaterThan(v, version.Current())
+			if err != nil {
+				return nil, fmt.Errorf("error parsing configuration list: invalid cniVersions entry %s at index %d: %w", v, i, err)
+			} else if !gt {
+				// Skip versions "greater" than this implementation of the spec
+				vs = append(vs, v)
+			}
+		}
+
+		// if cniVersion was already set, append it to the list for sorting.
+		if cniVersion != "" {
+			gt, err := version.GreaterThan(cniVersion, version.Current())
+			if err != nil {
+				return nil, fmt.Errorf("error parsing configuration list: invalid cniVersion %s: %w", cniVersion, err)
+			} else if !gt {
+				// ignore any versions higher than the current implemented spec version
+				vs = append(vs, cniVersion)
+			}
+		}
+		slices.SortFunc[[]string](vs, func(v1, v2 string) int {
+			if v1 == v2 {
+				return 0
+			}
+			if gt, _ := version.GreaterThan(v1, v2); gt {
+				return 1
+			}
+			return -1
+		})
+		if len(vs) > 0 {
+			cniVersion = vs[len(vs)-1]
 		}
 	}
 
+	readBool := func(key string) (bool, error) {
+		rawVal, ok := rawList[key]
+		if !ok {
+			return false, nil
+		}
+		if b, ok := rawVal.(bool); ok {
+			return b, nil
+		}
+
+		s, ok := rawVal.(string)
+		if !ok {
+			return false, fmt.Errorf("error parsing configuration list: invalid type %T for %s", rawVal, key)
+		}
+		s = strings.ToLower(s)
+		switch s {
+		case "false":
+			return false, nil
+		case "true":
+			return true, nil
+		}
+		return false, fmt.Errorf("error parsing configuration list: invalid value %q for %s", s, key)
+	}
+
+	disableCheck, err := readBool("disableCheck")
+	if err != nil {
+		return nil, err
+	}
+
+	disableGC, err := readBool("disableGC")
+	if err != nil {
+		return nil, err
+	}
+
+	loadOnlyInlinedPlugins, err := readBool("loadOnlyInlinedPlugins")
+	if err != nil {
+		return nil, err
+	}
+
 	list := &NetworkConfigList{
-		Name:         name,
-		DisableCheck: disableCheck,
-		CNIVersion:   cniVersion,
-		Bytes:        bytes,
+		Name:                   name,
+		DisableCheck:           disableCheck,
+		DisableGC:              disableGC,
+		LoadOnlyInlinedPlugins: loadOnlyInlinedPlugins,
+		CNIVersion:             cniVersion,
+		Bytes:                  confBytes,
 	}
 
 	var plugins []interface{}
 	plug, ok := rawList["plugins"]
-	if !ok {
-		return nil, fmt.Errorf("error parsing configuration list: no 'plugins' key")
+	// We can have a `plugins` list key in the main conf,
+	// We can also have `loadOnlyInlinedPlugins == true`
+	//
+	// If `plugins` is there, then `loadOnlyInlinedPlugins` can be true
+	//
+	// If plugins is NOT there, then `loadOnlyInlinedPlugins` cannot be true
+	//
+	// We have to have at least some plugins.
+	if !ok && loadOnlyInlinedPlugins {
+		return nil, fmt.Errorf("error parsing configuration list: `loadOnlyInlinedPlugins` is true, and no 'plugins' key")
+	} else if !ok && !loadOnlyInlinedPlugins {
+		return list, nil
 	}
+
 	plugins, ok = plug.([]interface{})
 	if !ok {
 		return nil, fmt.Errorf("error parsing configuration list: invalid 'plugins' type %T", plug)
@@ -124,24 +241,68 @@ func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) {
 		}
 		list.Plugins = append(list.Plugins, netConf)
 	}
-
 	return list, nil
 }
 
-func ConfListFromFile(filename string) (*NetworkConfigList, error) {
-	bytes, err := ioutil.ReadFile(filename)
+func NetworkConfFromFile(filename string) (*NetworkConfigList, error) {
+	bytes, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("error reading %s: %w", filename, err)
 	}
-	return ConfListFromBytes(bytes)
+
+	conf, err := NetworkConfFromBytes(bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	if !conf.LoadOnlyInlinedPlugins {
+		plugins, err := NetworkPluginConfsFromFiles(filepath.Dir(filename), conf.Name)
+		if err != nil {
+			return nil, err
+		}
+		conf.Plugins = append(conf.Plugins, plugins...)
+	}
+
+	if len(conf.Plugins) == 0 {
+		// Having 0 plugins for a given network is not necessarily a problem,
+		// but return as error for caller to decide, since they tried to load
+		return nil, fmt.Errorf("no plugin configs found")
+	}
+	return conf, nil
 }
 
+// Deprecated: This file format is no longer supported, use NetworkConfXXX and NetworkPluginXXX functions
+func ConfFromBytes(bytes []byte) (*NetworkConfig, error) {
+	return NetworkPluginConfFromBytes(bytes)
+}
+
+// Deprecated: This file format is no longer supported, use NetworkConfXXX and NetworkPluginXXX functions
+func ConfFromFile(filename string) (*NetworkConfig, error) {
+	bytes, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", filename, err)
+	}
+	return ConfFromBytes(bytes)
+}
+
+func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) {
+	return NetworkConfFromBytes(bytes)
+}
+
+func ConfListFromFile(filename string) (*NetworkConfigList, error) {
+	return NetworkConfFromFile(filename)
+}
+
+// ConfFiles simply returns a slice of all files in the provided directory
+// with extensions matching the provided set.
 func ConfFiles(dir string, extensions []string) ([]string, error) {
 	// In part, adapted from rkt/networking/podenv.go#listFiles
-	files, err := ioutil.ReadDir(dir)
+	files, err := os.ReadDir(dir)
 	switch {
 	case err == nil: // break
 	case os.IsNotExist(err):
+		// If folder not there, return no error - only return an
+		// error if we cannot read contents or there are no contents.
 		return nil, nil
 	default:
 		return nil, err
@@ -162,6 +323,7 @@ func ConfFiles(dir string, extensions []string) ([]string, error) {
 	return confFiles, nil
 }
 
+// Deprecated: This file format is no longer supported, use NetworkConfXXX and NetworkPluginXXX functions
 func LoadConf(dir, name string) (*NetworkConfig, error) {
 	files, err := ConfFiles(dir, []string{".conf", ".json"})
 	switch {
@@ -185,6 +347,15 @@ func LoadConf(dir, name string) (*NetworkConfig, error) {
 }
 
 func LoadConfList(dir, name string) (*NetworkConfigList, error) {
+	return LoadNetworkConf(dir, name)
+}
+
+// LoadNetworkConf looks at all the network configs in a given dir,
+// loads and parses them all, and returns the first one with an extension of `.conf`
+// that matches the provided network name predicate.
+func LoadNetworkConf(dir, name string) (*NetworkConfigList, error) {
+	// TODO this .conflist/.conf extension thing is confusing and inexact
+	// for implementors. We should pick one extension for everything and stick with it.
 	files, err := ConfFiles(dir, []string{".conflist"})
 	if err != nil {
 		return nil, err
@@ -192,7 +363,7 @@ func LoadConfList(dir, name string) (*NetworkConfigList, error) {
 	sort.Strings(files)
 
 	for _, confFile := range files {
-		conf, err := ConfListFromFile(confFile)
+		conf, err := NetworkConfFromFile(confFile)
 		if err != nil {
 			return nil, err
 		}
@@ -201,12 +372,13 @@ func LoadConfList(dir, name string) (*NetworkConfigList, error) {
 		}
 	}
 
-	// Try and load a network configuration file (instead of list)
+	// Deprecated: Try and load a network configuration file (instead of list)
 	// from the same name, then upconvert.
 	singleConf, err := LoadConf(dir, name)
 	if err != nil {
 		// A little extra logic so the error makes sense
-		if _, ok := err.(NoConfigsFoundError); len(files) != 0 && ok {
+		var ncfErr NoConfigsFoundError
+		if len(files) != 0 && errors.As(err, &ncfErr) {
 			// Config lists found but no config files found
 			return nil, NotFoundError{dir, name}
 		}
@@ -216,7 +388,8 @@ func LoadConfList(dir, name string) (*NetworkConfigList, error) {
 	return ConfListFromConf(singleConf)
 }
 
-func InjectConf(original *NetworkConfig, newValues map[string]interface{}) (*NetworkConfig, error) {
+// InjectConf takes a PluginConfig and inserts additional values into it, ensuring the result is serializable.
+func InjectConf(original *PluginConfig, newValues map[string]interface{}) (*PluginConfig, error) {
 	config := make(map[string]interface{})
 	err := json.Unmarshal(original.Bytes, &config)
 	if err != nil {
@@ -240,12 +413,14 @@ func InjectConf(original *NetworkConfig, newValues map[string]interface{}) (*Net
 		return nil, err
 	}
 
-	return ConfFromBytes(newBytes)
+	return NetworkPluginConfFromBytes(newBytes)
 }
 
 // ConfListFromConf "upconverts" a network config in to a NetworkConfigList,
 // with the single network as the only entry in the list.
-func ConfListFromConf(original *NetworkConfig) (*NetworkConfigList, error) {
+//
+// Deprecated: Non-conflist file formats are unsupported, use NetworkConfXXX and NetworkPluginXXX functions
+func ConfListFromConf(original *PluginConfig) (*NetworkConfigList, error) {
 	// Re-deserialize the config's json, then make a raw map configlist.
 	// This may seem a bit strange, but it's to make the Bytes fields
 	// actually make sense. Otherwise, the generated json is littered with

vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go

@@ -51,25 +51,34 @@ func DelegateAdd(ctx context.Context, delegatePlugin string, netconf []byte, exe
 // DelegateCheck calls the given delegate plugin with the CNI CHECK action and
 // JSON configuration
 func DelegateCheck(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error {
+	return delegateNoResult(ctx, delegatePlugin, netconf, exec, "CHECK")
+}
+
+func delegateNoResult(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec, verb string) error {
 	pluginPath, realExec, err := delegateCommon(delegatePlugin, exec)
 	if err != nil {
 		return err
 	}
 
-	// DelegateCheck will override the original CNI_COMMAND env from process with CHECK
-	return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("CHECK"), realExec)
+	return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs(verb), realExec)
 }
 
 // DelegateDel calls the given delegate plugin with the CNI DEL action and
 // JSON configuration
 func DelegateDel(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error {
-	pluginPath, realExec, err := delegateCommon(delegatePlugin, exec)
-	if err != nil {
-		return err
-	}
+	return delegateNoResult(ctx, delegatePlugin, netconf, exec, "DEL")
+}
 
-	// DelegateDel will override the original CNI_COMMAND env from process with DEL
-	return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("DEL"), realExec)
+// DelegateStatus calls the given delegate plugin with the CNI STATUS action and
+// JSON configuration
+func DelegateStatus(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error {
+	return delegateNoResult(ctx, delegatePlugin, netconf, exec, "STATUS")
+}
+
+// DelegateGC calls the given delegate plugin with the CNI GC action and
+// JSON configuration
+func DelegateGC(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error {
+	return delegateNoResult(ctx, delegatePlugin, netconf, exec, "GC")
 }
 
 // return CNIArgs used by delegation