Merge pull request #1470 from trozet/add_status_delegation

Adds support for CNI STATUS + other fixes for CNI Spec 1.1.0

.github/workflows/build.yml

@@ -4,18 +4,18 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go-version: [1.24.x]
         goarch: [386, amd64, arm, arm64, ppc64le, s390x]
         os: [ubuntu-latest] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Build
       env:

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/image-build.yml

@@ -6,14 +6,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
@@ -25,50 +25,72 @@ jobs:
 
       # note: disable sbom/provenance for now (gchr.io does not managed well yet)
       - name: Build container debug image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
           tags: ghcr.io/${{ github.repository }}:latest
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
-  build-amd64-thick:
-    name: Image build/amd64 thick plugin
+  build-thick:
+    name: Image thick plugin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build container image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false
-          tags: ghcr.io/${{ github.repository }}:latest-amd64-thick
+          tags: ghcr.io/${{ github.repository }}:latest-thick
           file: images/Dockerfile.thick
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@0.29.0
+        with:
+          image-ref: ghcr.io/${{ github.repository }}:latest-thick
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
 
   build-origin:
     name: Image build/origin
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Download OKD Builder Dockerfile
-        run: curl https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile -o images/okd-builder.Dockerfile
+        run: curl https://raw.githubusercontent.com/okd-project/images/main/builder/Dockerfile -o images/okd-builder.Dockerfile
 
       - name: Patch OKD Builder Dockerfile to workaround error
         run: sed -i -e "s/yum install -y yum-utils/rpm --import \/etc\/pki\/rpm-gpg\/*;yum install -y yum-utils/" images/okd-builder.Dockerfile
 
+      - name: Create root for builder
+        run: mkdir root
+
       - name: Organically build golang builder image
         run: docker build -t local/okdbuilder:latest -f images/okd-builder.Dockerfile .
 

.github/workflows/image-push-master.yml

@@ -1,24 +1,24 @@
 name: Image push for master
-on: 
+on:
   push:
     branches:
       - master
 env:
   image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick-amd64:
-    name: Image push thick image/amd64
+  push-thick:
+    name: Image push thick image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -26,7 +26,7 @@ jobs:
 
       - name: Push container image for thick plugin
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -34,21 +34,23 @@ jobs:
             ghcr.io/${{ github.repository }}:latest-thick
             ghcr.io/${{ github.repository }}:snapshot-thick
           file: images/Dockerfile.thick
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
   push-thin:
     name: Image push thin image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -56,7 +58,7 @@ jobs:
 
       - name: Push thin container image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -64,13 +66,13 @@ jobs:
             ghcr.io/${{ github.repository }}:latest
             ghcr.io/${{ github.repository }}:snapshot
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
       - name: Push thin container debug image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -78,7 +80,7 @@ jobs:
             ghcr.io/${{ github.repository }}:latest-debug
             ghcr.io/${{ github.repository }}:snapshot-debug
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
@@ -88,14 +90,14 @@ jobs:
 #    runs-on: ubuntu-latest
 #    steps:
 #      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v3
+#        uses: actions/checkout@v4
 #
 #      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v2
+#        uses: docker/setup-buildx-action@v3
 #
 #      - name: Login to GitHub Container Registry
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v2
+#        uses: docker/login-action@v3
 #        with:
 #          registry: ghcr.io
 #          username: ${{ github.repository_owner }}
@@ -103,7 +105,7 @@ jobs:
 #
 #      - name: Push container image
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v3
+#        uses: docker/build-push-action@v5
 #        with:
 #          context: .
 #          push: true

.github/workflows/image-push-release.yml

@@ -1,24 +1,24 @@
 name: Image push release
-on: 
+on:
   push:
     tags:
       - v*
 env:
   image-push-owner: 'k8snetworkplumbingwg'
 jobs:
-  push-thick-amd64:
-    name: Image push thick image/amd64
+  push-thick:
+    name: Image push thick image
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -26,7 +26,7 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/${{ github.repository }}
           flavor: |
@@ -34,7 +34,7 @@ jobs:
 
       - name: Push container image for thick plugin
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -42,21 +42,23 @@ jobs:
             ghcr.io/${{ github.repository }}:stable-thick
             ${{ steps.docker_meta.outputs.tags }}-thick
           file: images/Dockerfile.thick
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
 
   push-thin:
     name: Image push thin image/amd64
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -64,7 +66,7 @@ jobs:
 
       - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/${{ github.repository }}
           flavor: |
@@ -72,7 +74,7 @@ jobs:
 
       - name: Push thin container image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -80,13 +82,13 @@ jobs:
             ghcr.io/${{ github.repository }}:stable
             ${{ steps.docker_meta.outputs.tags }}
           file: images/Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
       - name: Push thin container debug image
         if: ${{ github.repository_owner == env.image-push-owner }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -94,7 +96,7 @@ jobs:
             ghcr.io/${{ github.repository }}:stable-debug
             ${{ steps.docker_meta.outputs.tags }}-debug
           file: images/Dockerfile.debug
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
           sbom: false
           provenance: false
 
@@ -104,14 +106,14 @@ jobs:
 #    runs-on: ubuntu-latest
 #    steps:
 #      - name: Check out code into the Go module directory
-#        uses: actions/checkout@v3
+#        uses: actions/checkout@v4
 #
 #      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v1
+#        uses: docker/setup-buildx-action@v3
 #
 #      - name: Login to GitHub Container Registry
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/login-action@v1
+#        uses: docker/login-action@v3
 #        with:
 #          registry: ghcr.io
 #          username: ${{ github.repository_owner }}
@@ -126,7 +128,7 @@ jobs:
 #
 #      - name: Push container image
 #        if: github.repository_owner == 'k8snetworkplumbingwg'
-#        uses: docker/build-push-action@v2
+#        uses: docker/build-push-action@v5
 #        with:
 #          context: .
 #          push: true

.github/workflows/kind-e2e.yml

@@ -25,32 +25,31 @@ jobs:
 #          - docker-file: images/Dockerfile
 #            cni-version: "1.0.0"
 #            multus-manifest: multus-daemonset.yml
+    env:
+      JOB_NAME: "${{ matrix.cni-version }}-${{ matrix.multus-manifest }}"
 
     if: >
       (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
         github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
 
       - name: Setup j2cli
         run: |
-          pip3 install --user --upgrade j2cli
-          j2 --version
-
-      - name: Setup registry
-        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+          sudo apt-get install -y j2cli
+          echo $(j2 --version)
 
       - name: Build latest-amd64
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           load: true
@@ -58,10 +57,6 @@ jobs:
           file: ${{ matrix.docker-file }}
           platforms: linux/amd64
 
-      # docker buildx push is failed due to https://github.com/docker/buildx/issues/94
-      - name: Push to local registry
-        run: docker push localhost:5000/multus:e2e
-
       - name: Get kind/kubectl/koko
         working-directory: ./e2e
         run: ./get_tools.sh
@@ -72,7 +67,7 @@ jobs:
 
       - name: Setup cluster
         working-directory: ./e2e
-        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} ./setup_cluster.sh
+        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} MULTUS_DOCKERFILE=none ./setup_cluster.sh
 
       - name: Test simple pod
         working-directory: ./e2e
@@ -90,8 +85,34 @@ jobs:
         working-directory: ./e2e
         run: ./test-default-route1.sh
 
+#      - name: Test DRA integration
+#        working-directory: ./e2e
+#        run: ./test-dra-integration.sh
+#
+
+      - name: Test subdirectory CNI chaining
+        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        working-directory: ./e2e
+        run: ./test-subdirectory-chaining.sh
+
+      - name: Test subdirectory CNI chaining with passthru CNI / auxiliaryCNIChainName
+        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        working-directory: ./e2e
+        run: ./test-subdirectory-chaining-passthru.sh
+
+      - name: Export kind logs
+        if: always()
+        run: |
+          mkdir -p /tmp/kind/logs
+          kind export logs /tmp/kind/logs -v 2147483647
+
+      - name: Upload kind logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
+          path: /tmp/kind/logs
+
       - name: cleanup cluster and registry
         run: |
           kind delete cluster
-          docker kill kind-registry
-          docker rm kind-registry

.github/workflows/release.yml

@@ -8,17 +8,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: 1.24.x
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v4
+        uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist

.github/workflows/stale-issues-prs.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'

.github/workflows/test.yml

@@ -4,17 +4,17 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
 
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run Revive Action by pulling pre-built image
       uses: docker://morphy/revive-action:v2

.gitignore

@@ -2,6 +2,7 @@
 bin/
 e2e/bin/
 e2e/yamls/
+e2e/repos/
 
 # GOPATH created by the build script
 gopath/

.travis.yml

@@ -1,116 +0,0 @@
-os: linux
-language: go
-# see https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments
-# for the detail
-# sudo: requried
-dist: bionic
-
-services:
-  - docker
-
-go:
-  - 1.13.x
-
-env:
-  global:
-    - GO111MODULE=on
-    - REGISTRY_USER=${REGISTRY_USER:-nfvpe}
-    - REGISTRY_PASS=${REGISTRY_PASS}
-    - REPOSITORY_NAME=${REPOSITORY_NAME}
-    - REPOSITORY_USER=${REPOSITORY_USER}
-    - DOCKER_CLI_EXPERIMENTAL="enabled"
-    - secure: "${REGISTRY_SECURE}"
-  jobs:
-    - TARGET=amd64
-    - TARGET=ppc64le
-
-before_install:
-  - if [ "${REPOSITORY_NAME}" = "" ]; then export REPOSITORY_NAME=multus; fi
-  - sudo apt-get update -qq
-  - go get github.com/mattn/goveralls
-
-install:
-  - go get -u golang.org/x/lint/golint
-
-before_script:
-  # Make gopath... to run golint/go fmt/go vet
-  # Suppress golint for fixing lint later.
-  - golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
-  - go fmt ./...
-  - go vet ./...
-#  - gocyclo -over 15 ./multus
-
-script:
-  - GOARCH="${TARGET}" ./hack/build-go.sh
-  - |
-    if [ "${TARGET}" == "amd64" ]; then
-      sudo env PATH=${PATH} ./scripts/test.sh
-      goveralls -coverprofile=coverage.out -service=travis-ci
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 .
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le -f Dockerfile.ppc64le .
-      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}-origin:latest -f Dockerfile.openshift .
-    fi
-
-deploy:
-  # Release on versioned tag (e.g. v1.0)
-  - provider: script
-    #cleanup: false
-    script: curl -sL https://git.io/goreleaser
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TARGET = amd64 && $TRAVIS_TAG =~ ^v[0-9].*$ && ! -z $GITHUB_TOKEN && $TRAVIS_OS_NAME = linux"
-  # Push images to Dockerhub on tag
-  - provider: script
-    cleanup: false
-    script: >
-      bash -c '
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
-      export DOCKER_CLI_EXPERIMENTAL="enabled";
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
-      echo done'
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TRAVIS_TAG =~ ^v[0-9].*$ && -n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
-  # Push images to Dockerhub on merge to master
-  - provider: script
-    on:
-      branch: master
-      condition: "-n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
-    script: >
-      bash -c '
-      docker tag ${REPOSITORY_USER}/:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
-      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
-      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
-      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
-      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
-      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
-      echo done'

Makefile

@@ -0,0 +1,14 @@
+
+
+.PHONY: deps-update
+deps-update: ; $(info  Updating dependencies...) @ ## Update dependencies
+	go mod tidy
+	go mod vendor
+
+PHONY: build test
+
+build:
+	./hack/build-go.sh
+
+test:
+	sudo ./hack/test-go.sh

README.md

@@ -24,10 +24,10 @@ Here's an illustration of the network interfaces attached to a pod, as provision
 
 The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugin, refer to our [quick-start guide](docs/quickstart.md).
 
-Clone this GitHub repository, and apply a daemonset which installs Multus using `kubectl`. From the root directory of the clone, apply the daemonset YAML file:
+To use latest features try command below which applies a daemonset and installs thick Multus using `kubectl`:
 
 ```
-cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
 
 This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
@@ -39,7 +39,7 @@ With the multus 4.0 release, we introduce a new client/server-style plugin deplo
 We recommend using the thick plugin in most environments, but if you wish to run the thin plugin, or are in a resource-constrained environment, you may do so with:
 
 ```
-cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 ```
 
 ## Additional Installation Options
@@ -62,3 +62,5 @@ In addition to the [quick-start guide](docs/quickstart.md), you may:
 ## Contact Us
 
 For any questions about Multus CNI, open up a GitHub issue or feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/).
+
+To be invited, use [this slack invite link](https://join.slack.com/t/npwg-team/shared_invite/zt-1u2vmsn2b-tKdOokdPY73zn9B32JoAOg).

cmd/cert-approver/main.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2023 Network Plumbing Working Group
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is Kubernetes controller which approves CSR submitted by multus.
+// This command is required only if multus runs with per-node certificate.
+package main
+
+// Note: cert-approver should be simple, just approve multus' CSR, hence
+// this go code should not have any dependencies from pkg/, if possible,
+// to keep its code simplicity.
+import (
+	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"os/signal"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/certificate/csr"
+	"k8s.io/client-go/util/workqueue"
+)
+
+// CertController object
+type CertController struct {
+	clientset          kubernetes.Interface
+	queue              workqueue.RateLimitingInterface
+	informer           cache.SharedIndexInformer
+	broadcaster        record.EventBroadcaster
+	recorder           record.EventRecorder
+	commonNamePrefixes string
+}
+
+const (
+	maxDuration                = time.Hour * 24 * 365
+	resyncPeriod time.Duration = time.Second * 3600 // resync every one hour, default is 10 hour
+	maxRetries                 = 5
+)
+
+var (
+	// ControllerName provides controller name
+	ControllerName = "csr-approver"
+	// NamePrefix specifies which name in certification request should be target to approve
+	NamePrefix = "system:multus"
+	// Organization specifies which org in certification request should be target to approve
+	Organization = []string{"system:multus"}
+	// Groups specifies which group in certification request should be target to approve
+	Groups = sets.New[string]("system:nodes", "system:multus", "system:authenticated")
+	// UserPrefixes specifies which name prefix in certification request should be target to approve
+	UserPrefixes = sets.New[string]("system:node", NamePrefix)
+	// Usages specifies which usage in certification request should be target to approve
+	Usages = sets.New[certificatesv1.KeyUsage](
+		certificatesv1.UsageDigitalSignature,
+		certificatesv1.UsageClientAuth)
+)
+
+// NewCertController creates certcontroller
+func NewCertController() (*CertController, error) {
+	var clientset kubernetes.Interface
+	// setup Kubernetes API client
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientset, err = kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	informer := cache.NewSharedIndexInformer(
+		cache.NewListWatchFromClient(
+			clientset.CertificatesV1().RESTClient(),
+			"certificatesigningrequests", corev1.NamespaceAll, fields.Everything()),
+		&certificatesv1.CertificateSigningRequest{},
+		resyncPeriod,
+		nil)
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientset.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "cert-approver"})
+	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+	c := &CertController{
+		clientset:          clientset,
+		informer:           informer,
+		queue:              queue,
+		commonNamePrefixes: NamePrefix,
+		broadcaster:        broadcaster,
+		recorder:           recorder,
+	}
+
+	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			if csr, ok := obj.(*certificatesv1.CertificateSigningRequest); ok {
+				if c.filterCSR(csr) {
+					key, err := cache.MetaNamespaceKeyFunc(obj)
+					if err == nil {
+						queue.Add(key)
+					}
+				}
+			}
+		},
+	})
+
+	return c, nil
+}
+
+// Run starts controller
+func (c *CertController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting cert approver")
+
+	go c.informer.Run(stopCh)
+	if !cache.WaitForCacheSync(stopCh, c.HasSynced) {
+		utilruntime.HandleError(fmt.Errorf("Timed out waiting for caches to sync"))
+		return
+	}
+
+	klog.Info("cert approver synced and ready")
+	wait.Until(c.runWorker, time.Second, stopCh)
+}
+
+// HasSynced is required for the cache.Controller interface.
+func (c *CertController) HasSynced() bool {
+	return c.informer.HasSynced()
+}
+
+// LastSyncResourceVersion is required for the cache.Controller interface.
+func (c *CertController) LastSyncResourceVersion() string {
+	return c.informer.LastSyncResourceVersion()
+}
+
+func (c *CertController) runWorker() {
+	for c.processNextItem() {
+		// continue looping
+	}
+}
+
+func (c *CertController) processNextItem() bool {
+	// Wait until there is a new item in the working queue
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	// Tell the queue that we are done with processing this key. This unblocks the key for other workers
+	// This allows safe parallel processing because two pods with the same key are never processed in
+	// parallel.
+	defer c.queue.Done(key)
+
+	// Invoke the method containing the business logic
+	err := c.processItem(key.(string))
+	// Handle the error if something went wrong during the execution of the business logic
+	c.handleErr(err, key)
+	return true
+
+}
+
+// handleErr checks if an error happened and makes sure we will retry later.
+func (c *CertController) handleErr(err error, key interface{}) {
+	if err == nil {
+		// Forget about the #AddRateLimited history of the key on every successful synchronization.
+		// This ensures that future processing of updates for this key is not delayed because of
+		// an outdated error history.
+		c.queue.Forget(key)
+		return
+	}
+
+	// This controller retries 5 times if something goes wrong. After that, it stops trying.
+	if c.queue.NumRequeues(key) < maxRetries {
+		klog.Infof("Error syncing csr %s: %v", key, err)
+		// Re-enqueue the key rate limited. Based on the rate limiter on the
+		// queue and the re-enqueue history, the key will be processed later again.
+		c.queue.AddRateLimited(key)
+		return
+	}
+
+	c.queue.Forget(key)
+	// Report to an external entity that, even after several retries, we could not successfully process this key
+	utilruntime.HandleError(err)
+	klog.Infof("Dropping csr %q out of the queue: %v", key, err)
+}
+
+func (c *CertController) processItem(key string) error {
+	startTime := time.Now()
+
+	obj, _, err := c.informer.GetIndexer().GetByKey(key)
+	if err != nil {
+		return fmt.Errorf("Error fetching object with key %s from store: %v", key, err)
+	}
+
+	req, _ := obj.(*certificatesv1.CertificateSigningRequest)
+
+	nodeName := "unknown"
+	defer func() {
+		klog.Infof("Finished syncing CSR %s for %s node in %v", req.Name, nodeName, time.Since(startTime))
+	}()
+
+	if len(req.Status.Certificate) > 0 {
+		klog.V(5).Infof("CSR %s is already signed", req.Name)
+		return nil
+	}
+
+	if isApprovedOrDenied(&req.Status) {
+		klog.V(5).Infof("CSR %s is already approved/denied", req.Name)
+		return nil
+	}
+
+	csrPEM, _ := pem.Decode(req.Spec.Request)
+	if csrPEM == nil {
+		return fmt.Errorf("failed to PEM-parse the CSR block in .spec.request: no CSRs were found")
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse the CSR bytes: %v", err)
+	}
+
+	i := strings.LastIndex(req.Spec.Username, ":")
+	if i == -1 || i == len(req.Spec.Username)-1 {
+		return fmt.Errorf("failed to parse the username: %s", req.Spec.Username)
+	}
+
+	ctx := context.Background()
+	prefix := req.Spec.Username[:i]
+	nodeName = req.Spec.Username[i+1:]
+	if !UserPrefixes.Has(prefix) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by an unexpected user: %q", req.Name, req.Spec.Username))
+	}
+
+	if errs := validation.IsDNS1123Subdomain(nodeName); len(errs) != 0 {
+		return c.denyCSR(ctx, req, fmt.Sprintf("extracted node name %q is not a valid DNS subdomain %v", nodeName, errs))
+	}
+
+	if usages := sets.New[certificatesv1.KeyUsage](req.Spec.Usages...); !usages.Equal(Usages) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with unexpected usages: %v", req.Name, usages.UnsortedList()))
+	}
+
+	if !Groups.HasAll(req.Spec.Groups...) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by a user with unexpected groups: %v", req.Name, req.Spec.Groups))
+	}
+
+	expectedSubject := fmt.Sprintf("%s:%s", c.commonNamePrefixes, nodeName)
+	if x509CSR.Subject.CommonName != expectedSubject {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's commonName to be %q, but it is %q", expectedSubject, x509CSR.Subject.CommonName))
+	}
+
+	if !reflect.DeepEqual(x509CSR.Subject.Organization, Organization) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's organization to be %v, but it is %v", Organization, x509CSR.Subject.Organization))
+	}
+
+	if req.Spec.ExpirationSeconds == nil {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created without specyfying the expirationSeconds", req.Name))
+	}
+
+	if csr.ExpirationSecondsToDuration(*req.Spec.ExpirationSeconds) > maxDuration {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with invalid expirationSeconds value: %d", req.Name, *req.Spec.ExpirationSeconds))
+	}
+
+	return c.approveCSR(ctx, req)
+}
+
+// CSR specific functions
+
+func (c *CertController) filterCSR(csr *certificatesv1.CertificateSigningRequest) bool {
+	nsName := types.NamespacedName{Namespace: csr.Namespace, Name: csr.Name}
+	csrPEM, _ := pem.Decode(csr.Spec.Request)
+	if csrPEM == nil {
+		klog.Errorf("Failed to PEM-parse the CSR block in .spec.request: no CSRs were found in %s", nsName)
+		return false
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		klog.Errorf("Failed to parse the CSR .spec.request of %q: %v", nsName, err)
+		return false
+	}
+
+	return strings.HasPrefix(x509CSR.Subject.CommonName, c.commonNamePrefixes) &&
+		csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName
+}
+
+func (c *CertController) approveCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateApproved,
+			Status:  corev1.ConditionTrue,
+			Reason:  "AutoApproved",
+			Message: fmt.Sprintf("Auto-approved CSR %q", csr.Name),
+		})
+
+	c.recorder.Eventf(csr, corev1.EventTypeNormal, "CSRApproved", "CSR %q has been approved by %s", csr.Name, ControllerName)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func (c *CertController) denyCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, message string) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateDenied,
+			Status:  corev1.ConditionTrue,
+			Reason:  "CSRDenied",
+			Message: message,
+		},
+	)
+
+	c.recorder.Eventf(csr, corev1.EventTypeWarning, "CSRDenied", "The CSR %q has been denied by: %s", csr.Name, ControllerName, message)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().Update(ctx, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func isApprovedOrDenied(status *certificatesv1.CertificateSigningRequestStatus) bool {
+	for _, c := range status.Conditions {
+		if c.Type == certificatesv1.CertificateApproved || c.Type == certificatesv1.CertificateDenied {
+			return true
+		}
+	}
+	return false
+}
+
+func main() {
+	klog.Infof("starting cert-approver")
+
+	// Start watching for pod creations
+	certController, err := NewCertController()
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+	go certController.Run(stopCh)
+
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+}

cmd/install_multus/main.go

@@ -53,4 +53,15 @@ func main() {
 	}
 
 	fmt.Printf("multus %s copy succeeded!\n", multusFileName)
+
+	// Copy the passthru CNI
+	passthruPath := "/usr/src/multus-cni/bin/passthru"
+	err = cmdutils.CopyFileAtomic(passthruPath, *destDir, fmt.Sprintf("%s.temp", "passthru"), "passthru")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("passthru cni %s copy succeeded!\n", passthruPath)
+
 }

cmd/kubeconfig_generator/main.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary submit CSR for kube controll access for multus thin plugin
+// and generate Kubeconfig
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"text/template"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+var kubeConfigTemplate = `apiVersion: v1
+clusters:
+  - cluster:
+      certificate-authority-data: {{.CADATA}}
+      server: {{.K8S_APISERVER}}
+    name: default-cluster
+contexts:
+  - context:
+      cluster: default-cluster
+      namespace: default
+      user: default-auth
+    name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+  - name: default-auth
+    user:
+      client-certificate: {{.CERTDIR}}/multus-client-current.pem
+      client-key: {{.CERTDIR}}/multus-client-current.pem
+`
+
+func main() {
+	certDir := pflag.StringP("certdir", "", "/tmp", "specify cert directory")
+	bootstrapConfig := pflag.StringP("bootstrap-config", "", "/tmp/kubeconfig", "specify bootstrap kubernetes config")
+	kubeconfigPathRaw := pflag.StringP("kubeconfig", "", "/run/multus/kubeconfig", "specify output kubeconfig path")
+	certDurationString := pflag.StringP("cert-duration", "", "10m", "specify certificate duration")
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	kubeconfigPath, err := filepath.Abs(*kubeconfigPathRaw)
+	if err != nil {
+		klog.Fatalf("illegal path %s in kubeconfigPath %s: %v", kubeconfigPath, *kubeconfigPathRaw, err)
+	}
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	// check variables
+	if _, err := os.Stat(*bootstrapConfig); err != nil {
+		klog.Fatalf("failed to read bootstrap config %q", *bootstrapConfig)
+	}
+	st, err := os.Stat(*certDir)
+	if err != nil {
+		klog.Fatalf("failed to find cert directory %q", *certDir)
+	}
+	if !st.IsDir() {
+		klog.Fatalf("cert directory %q is not directory", *certDir)
+	}
+	certDuration, err := time.ParseDuration(*certDurationString)
+	if err != nil {
+		klog.Fatalf("failed to parse duration %q: %v", *certDurationString, err)
+	}
+
+	nodeName := os.Getenv("MULTUS_NODE_NAME")
+	if nodeName == "" {
+		klog.Fatalf("cannot identify node name from MULTUS_NODE_NAME env variables")
+	}
+
+	// retrieve API server from bootstrapConfig()
+	config, err := clientcmd.BuildConfigFromFlags("", *bootstrapConfig)
+	if err != nil {
+		klog.Fatalf("cannot get in-cluster config: %v", err)
+	}
+	apiServer := fmt.Sprintf("%s%s", config.Host, config.APIPath)
+	caData := base64.StdEncoding.EncodeToString(config.CAData)
+
+	// run certManager to create certification
+	if _, err = k8sclient.PerNodeK8sClient(nodeName, *bootstrapConfig, certDuration, *certDir); err != nil {
+		klog.Fatalf("failed to start cert manager: %v", err)
+	}
+
+	fp, err := os.OpenFile(kubeconfigPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		klog.Fatalf("cannot create kubeconfig file %q: %v", kubeconfigPath, err)
+	}
+
+	// render kubeconfig
+	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
+	if err != nil {
+		klog.Fatalf("template parse error: %v", err)
+	}
+	templateData := map[string]string{
+		"CADATA":        caData,
+		"CERTDIR":       *certDir,
+		"K8S_APISERVER": apiServer,
+	}
+	// genearate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		klog.Fatalf("cannot create kubeconfig: %v", err)
+	}
+	if err = fp.Close(); err != nil {
+		klog.Fatalf("cannot save kubeconfig: %v", err)
+	}
+
+	klog.Infof("kubeconfig %q is saved", kubeconfigPath)
+
+	// wait for signal
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+	klog.Infof("signal received. remove kubeconfig %q and quit.", kubeconfigPath)
+	err = os.Remove(kubeconfigPath)
+	if err != nil {
+		klog.Errorf("failed to remove kubeconfig %q: %v", kubeconfigPath, err)
+	}
+}

cmd/multus-daemon/main.go

@@ -23,10 +23,12 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/signal"
 	"os/user"
 	"path/filepath"
+	"sync"
+	"syscall"
 
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
@@ -34,6 +36,7 @@ import (
 	srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -53,96 +56,91 @@ func main() {
 		os.Exit(4)
 	}
 
-	configWatcherStopChannel := make(chan struct{})
-	configWatcherDoneChannel := make(chan struct{})
-	serverStopChannel := make(chan struct{})
-	serverDoneChannel := make(chan struct{})
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
 
 	daemonConf, err := cniServerConfig(*configFilePath)
 	if err != nil {
+		logging.Panicf("startMultusDaemon failed to load the CNI server configuration: %v", err)
 		os.Exit(1)
 	}
 
-	if err := startMultusDaemon(daemonConf, serverStopChannel, serverDoneChannel); err != nil {
-		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
-		os.Exit(3)
-	}
-
 	multusConf, err := config.ParseMultusConfig(*configFilePath)
 	if err != nil {
 		logging.Panicf("startMultusDaemon failed to load the multus configuration: %v", err)
 		os.Exit(1)
 	}
 
-	// Generate multus CNI config from current CNI config
+	logging.Verbosef("multus-daemon started")
+
+	if multusConf.ReadinessIndicatorFile != "" {
+		// Check readinessindicator file before daemon launch
+		logging.Verbosef("Readiness Indicator file check")
+		if err := types.GetReadinessIndicatorFile(multusConf.ReadinessIndicatorFile); err != nil {
+			_ = logging.Errorf("have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", multusConf.ReadinessIndicatorFile, err)
+			os.Exit(1)
+		}
+		logging.Verbosef("Readiness Indicator file check done!")
+	}
+
+	var configManager *config.Manager
+	var ignoreReadinessIndicator bool
 	if multusConf.MultusConfigFile == "auto" {
 		if multusConf.CNIVersion == "" {
 			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
 		}
 
-		multusConf.SocketDir = daemonConf.SocketDir
-
-		var configManager *config.Manager
-		if multusConf.MultusMasterCni == "" {
-			configManager, err = config.NewManager(*multusConf, multusConf.MultusAutoconfigDir, multusConf.ForceCNIVersion)
-		} else {
-			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
-				*multusConf, multusConf.MultusAutoconfigDir, multusConf.MultusMasterCni, multusConf.ForceCNIVersion)
-		}
+		// Generate multus CNI config from current CNI config
+		configManager, err = config.NewManager(*multusConf)
 		if err != nil {
 			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
 			os.Exit(2)
 		}
-
-		if multusConf.OverrideNetworkName {
-			if err := configManager.OverrideNetworkName(); err != nil {
-				_ = logging.Errorf("could not override the network name: %v", err)
-			}
-		}
-
-		generatedMultusConfig, err := configManager.GenerateConfig()
-		if err != nil {
-			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
-		}
-		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
-
-		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
-			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
-		}
-
-		go func(stopChannel chan<- struct{}, doneChannel chan<- struct{}) {
-			if err := configManager.MonitorPluginConfiguration(configWatcherStopChannel, doneChannel); err != nil {
-				_ = logging.Errorf("error watching file: %v", err)
-			}
-		}(configWatcherStopChannel, configWatcherDoneChannel)
-
-		<-configWatcherDoneChannel
+		// ConfigManager watches the readiness indicator file (if configured)
+		// and exits the daemon when that is removed. The CNIServer does
+		// not need to re-do that check every CNI operation
+		ignoreReadinessIndicator = true
 	} else {
 		if err := copyUserProvidedConfig(multusConf.MultusConfigFile, multusConf.CniConfigDir); err != nil {
 			logging.Errorf("failed to copy the user provided configuration %s: %v", multusConf.MultusConfigFile, err)
 		}
 	}
 
-	serverDone := false
-	configWatcherDone := false
-	for {
-		select {
-		case <-configWatcherDoneChannel:
-			logging.Verbosef("ConfigWatcher done")
-			configWatcherDone = true
-		case <-serverDoneChannel:
-			logging.Verbosef("multus-server done.")
-			serverDone = true
-		}
+	if err := startMultusDaemon(ctx, daemonConf, ignoreReadinessIndicator); err != nil {
+		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
+		os.Exit(3)
+	}
 
-		if serverDone && configWatcherDone {
-			return
+	// Wait until daemon ready
+	logging.Verbosef("API readiness check")
+	if api.WaitUntilAPIReady(daemonConf.SocketDir) != nil {
+		logging.Panicf("failed to ready multus-daemon socket: %v", err)
+		os.Exit(1)
+	}
+	logging.Verbosef("API readiness check done!")
+
+	signalCh := make(chan os.Signal, 16)
+	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		for sig := range signalCh {
+			logging.Verbosef("caught %v, stopping...", sig)
+			cancel()
+		}
+	}()
+
+	var wg sync.WaitGroup
+	if configManager != nil {
+		if err := configManager.Start(ctx, &wg); err != nil {
+			_ = logging.Errorf("failed to start config manager: %v", err)
+			os.Exit(3)
 		}
 	}
-	// never reached
+
+	wg.Wait()
+	logging.Verbosef("multus daemon is exited")
 }
 
-func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}, done chan struct{}) error {
+func startMultusDaemon(ctx context.Context, daemonConfig *srv.ControllerNetConf, ignoreReadinessIndicator bool) error {
 	if user, err := user.Current(); err != nil || user.Uid != "0" {
 		return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
 	}
@@ -151,17 +149,17 @@ func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}
 		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
 	}
 
-	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents)
+	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents, ignoreReadinessIndicator)
 	if err != nil {
 		return fmt.Errorf("failed to create the server: %v", err)
 	}
 
 	if daemonConfig.MetricsPort != nil {
-		go utilwait.Until(func() {
+		go utilwait.UntilWithContext(ctx, func(_ context.Context) {
 			http.Handle("/metrics", promhttp.Handler())
 			logging.Debugf("metrics port: %d", *daemonConfig.MetricsPort)
 			logging.Debugf("metrics: %s", http.ListenAndServe(fmt.Sprintf(":%d", *daemonConfig.MetricsPort), nil))
-		}, 0, stopCh)
+		}, 0)
 	}
 
 	l, err := srv.GetListener(api.SocketPath(daemonConfig.SocketDir))
@@ -169,23 +167,23 @@ func startMultusDaemon(daemonConfig *srv.ControllerNetConf, stopCh chan struct{}
 		return fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(daemonConfig.SocketDir), err)
 	}
 
-	server.SetKeepAlivesEnabled(false)
+	server.Start(ctx, l)
+
 	go func() {
-		utilwait.Until(func() {
-			logging.Debugf("open for business")
-			if err := server.Serve(l); err != nil {
-				utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
-			}
-		}, 0, stopCh)
-		server.Shutdown(context.TODO())
-		close(done)
+		<-ctx.Done()
+		server.Shutdown(context.Background())
 	}()
 
 	return nil
 }
 
 func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
-	configFileContents, err := os.ReadFile(configFilePath)
+	path, err := filepath.Abs(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("illegal path %s in server config path %s: %w", path, configFilePath, err)
+	}
+
+	configFileContents, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
@@ -193,9 +191,14 @@ func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
 }
 
 func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
-	srcFile, err := os.Open(multusConfigPath)
+	path, err := filepath.Abs(multusConfigPath)
 	if err != nil {
-		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+		return fmt.Errorf("illegal path %s in multusConfigPath %s: %w", path, multusConfigPath, err)
+	}
+
+	srcFile, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", path, err)
 	}
 
 	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)

cmd/multus-shim/main.go

@@ -44,15 +44,23 @@ func main() {
 		return
 	}
 
-	skel.PluginMain(
-		func(args *skel.CmdArgs) error {
-			return api.CmdAdd(args)
-		},
-		func(args *skel.CmdArgs) error {
-			return api.CmdCheck(args)
-		},
-		func(args *skel.CmdArgs) error {
-			return api.CmdDel(args)
+	skel.PluginMainFuncs(
+		skel.CNIFuncs{
+			Add: func(args *skel.CmdArgs) error {
+				return api.CmdAdd(args)
+			},
+			Check: func(args *skel.CmdArgs) error {
+				return api.CmdCheck(args)
+			},
+			Del: func(args *skel.CmdArgs) error {
+				return api.CmdDel(args)
+			},
+			GC: func(args *skel.CmdArgs) error {
+				return api.CmdGC(args)
+			},
+			Status: func(args *skel.CmdArgs) error {
+				return api.CmdStatus(args)
+			},
 		},
 		cniversion.All, "meta-plugin that delegates to other CNI plugins")
 }

cmd/multus/main.go

@@ -43,17 +43,27 @@ func main() {
 		return
 	}
 
-	skel.PluginMain(
-		func(args *skel.CmdArgs) error {
-			result, err := multus.CmdAdd(args, nil, nil)
-			if err != nil {
-				return err
-			}
-			return result.Print()
+	skel.PluginMainFuncs(
+		skel.CNIFuncs{
+			Add: func(args *skel.CmdArgs) error {
+				result, err := multus.CmdAdd(args, nil, nil)
+				if err != nil {
+					return err
+				}
+				return result.Print()
+			},
+			Del: func(args *skel.CmdArgs) error {
+				return multus.CmdDel(args, nil, nil)
+			},
+			Check: func(args *skel.CmdArgs) error {
+				return multus.CmdCheck(args, nil, nil)
+			},
+			GC: func(args *skel.CmdArgs) error {
+				return multus.CmdGC(args, nil, nil)
+			},
+			Status: func(args *skel.CmdArgs) error {
+				return multus.CmdStatus(args, nil, nil)
+			},
 		},
-		func(args *skel.CmdArgs) error {
-			return multus.CmdCheck(args, nil, nil)
-		},
-		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
 		cniversion.All, "meta-plugin that delegates to other CNI plugins")
 }

cmd/passthru-cni/main.go

@@ -0,0 +1,58 @@
+// Package: passthru-cni
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniTypes "github.com/containernetworking/cni/pkg/types"
+	current "github.com/containernetworking/cni/pkg/types/100"
+	cniVersion "github.com/containernetworking/cni/pkg/version"
+)
+
+// NetConf is a CNI configuration structure
+type NetConf struct {
+	cniTypes.NetConf
+}
+
+func main() {
+	skel.PluginMain(
+		cmdAdd,
+		nil,
+		cmdDel,
+		cniVersion.PluginSupports("0.3.0", "0.3.1", "0.4.0", "1.0.0", "1.1.0"),
+		"Passthrough CNI Plugin v1.0",
+	)
+}
+
+func cmdAdd(args *skel.CmdArgs) error {
+	n, err := loadNetConf(args.StdinData)
+	if err != nil {
+		return fmt.Errorf("passthru cni: error parsing CNI configuration: %s", err)
+	}
+
+	// Create an empty but valid CNI result
+	result := &current.Result{
+		CNIVersion: n.CNIVersion,
+		Interfaces: []*current.Interface{},
+		IPs:        []*current.IPConfig{},
+		Routes:     []*cniTypes.Route{},
+		DNS:        cniTypes.DNS{},
+	}
+
+	return cniTypes.PrintResult(result, n.CNIVersion)
+}
+
+func cmdDel(_ *skel.CmdArgs) error {
+	// Nothing to do for DEL command, just return nil
+	return nil
+}
+
+func loadNetConf(bytes []byte) (*NetConf, error) {
+	n := &NetConf{}
+	if err := json.Unmarshal(bytes, n); err != nil {
+		return nil, fmt.Errorf("passthru cni: failed to load netconf: %s", err)
+	}
+	return n, nil
+}

cmd/thin_entrypoint/main.go

@@ -21,7 +21,7 @@ import (
 	b64 "encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
@@ -29,9 +29,11 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/libcni"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/spf13/pflag"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
 )
 
 // Options stores command line options
@@ -41,6 +43,7 @@ type Options struct {
 	CNIVersion               string
 	MultusConfFile           string
 	MultusBinFile            string // may be hidden or remove?
+	MultusCNIConfDir         string
 	SkipMultusBinaryCopy     bool
 	MultusKubeConfigFileHost string
 	MultusMasterCNIFileName  string
@@ -57,6 +60,7 @@ type Options struct {
 	AdditionalBinDir         string
 	ForceCNIVersion          bool
 	SkipTLSVerify            bool
+	SkipMultusConfWatch      bool
 }
 
 const (
@@ -72,6 +76,7 @@ func (o *Options) addFlags() {
 	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
 	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
 	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
+	fs.StringVar(&o.MultusCNIConfDir, "multus-cni-conf-dir", "/host/etc/cni/multus/net.d", "multus specific CNI config directory")
 	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")
 
 	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
@@ -83,7 +88,8 @@ func (o *Options) addFlags() {
 	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
 	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
 	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
+	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
 	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
@@ -138,18 +144,56 @@ contexts:
 current-context: multus-context
 `
 
-func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
-	// check file exists
-	if _, err := os.Stat(serviceAccountTokenFile); err != nil {
-		return nil, fmt.Errorf("service account token is not found: %v", err)
+func getFileAndHash(filepath string) ([]byte, []byte, error) {
+	if _, err := os.Stat(filepath); err != nil {
+		return nil, nil, fmt.Errorf("file %s not found: %v", filepath, err)
 	}
-	if _, err := os.Stat(serviceAccountCAFile); err != nil {
-		return nil, fmt.Errorf("service account ca is not found: %v", err)
+	content, err := os.ReadFile(filepath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot read %s file: %v", filepath, err)
+	}
+
+	hash := sha256.New()
+	hash.Write(content)
+	return content, hash.Sum(nil), nil
+}
+
+func (o *Options) createKubeConfig(prevCAHash, prevSATokenHash []byte) ([]byte, []byte, error) {
+	caFileByte, caHash, err := getFileAndHash(serviceAccountCAFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	saTokenByte, saTokenHash, err := getFileAndHash(serviceAccountTokenFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caUnchanged := prevCAHash != nil && bytes.Equal(prevCAHash, caHash)
+	saUnchanged := prevSATokenHash != nil && bytes.Equal(prevSATokenHash, saTokenHash)
+
+	if o.SkipTLSVerify {
+		if saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	} else {
+		if caUnchanged && saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	}
+
+	if prevSATokenHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("CA (%v) or SA token (%v) changed - recreating kubeconfig\n", !caUnchanged, !saUnchanged)
 	}
 
 	// create multus.d directory
 	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
-		return nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+		return nil, nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+	}
+
+	// create multus cni conf directory
+	if err := os.MkdirAll(o.MultusCNIConfDir, 0755); err != nil {
+		return nil, nil, fmt.Errorf("cannot create multus-cni-conf-dir(%s) directory: %v", o.MultusCNIConfDir, err)
 	}
 
 	// get Kubernetes service protocol/host/port
@@ -166,69 +210,49 @@ func (o *Options) createKubeConfig(currentFileHash []byte) ([]byte, error) {
 		tlsConfig = "insecure-skip-tls-verify: true"
 	} else {
 		// create tlsConfig by service account CA file
-		caFileByte, err := os.ReadFile(serviceAccountCAFile)
-		if err != nil {
-			return nil, fmt.Errorf("cannot read service account ca file: %v", err)
-		}
 		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
 		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
 	}
 
-	saTokenByte, err := os.ReadFile(serviceAccountTokenFile)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read service account token file: %v", err)
-	}
-
 	// create kubeconfig by template and replace it by atomic
 	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
 	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
 	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
+		return nil, nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
 	}
 
 	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
 	if err != nil {
-		return nil, fmt.Errorf("template parse error: %v", err)
+		return nil, nil, fmt.Errorf("template parse error: %v", err)
 	}
 	templateData := map[string]string{
-		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
+		"KubeConfigHost":          fmt.Sprintf("%s://%s", kubeProtocol, net.JoinHostPort(kubeHost, kubePort)),
 		"KubeServerTLS":           tlsConfig,
 		"KubeServiceAccountToken": string(saTokenByte),
 	}
 
-	// Prepare
-	hash := sha256.New()
-	writer := io.MultiWriter(hash, fp)
-
-	// genearate kubeconfig from template
-	if err = templateKubeconfig.Execute(writer, templateData); err != nil {
-		return nil, fmt.Errorf("cannot create kubeconfig: %v", err)
+	// generate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		return nil, nil, fmt.Errorf("cannot create kubeconfig: %v", err)
 	}
 
 	if err := fp.Sync(); err != nil {
 		os.Remove(fp.Name())
-		return nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
+		return nil, nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
 	}
 	if err := fp.Close(); err != nil {
 		os.Remove(fp.Name())
-		return nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
-	}
-
-	newFileHash := hash.Sum(nil)
-	if currentFileHash != nil && bytes.Compare(newFileHash, currentFileHash) == 0 {
-		fmt.Printf("kubeconfig is same, not copy\n")
-		os.Remove(fp.Name())
-		return currentFileHash, nil
+		return nil, nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
 	}
 
 	// replace file with tempfile
 	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
-		return nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
+		return nil, nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
 	}
 
 	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
-	return newFileHash, nil
+	return caHash, saTokenHash, nil
 }
 
 const multusConflistTemplate = `{
@@ -249,6 +273,8 @@ const multusConflistTemplate = `{
             .LogFileConfig
         }}{{
             .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
         }}{{
             .ReadinessIndicatorFileConfig
         }}
@@ -277,6 +303,8 @@ const multusConfTemplate = `{
             .LogFileConfig
         }}{{
             .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
         }}{{
             .ReadinessIndicatorFileConfig
         }}
@@ -287,27 +315,57 @@ const multusConfTemplate = `{
 }
 `
 
-func (o *Options) createMultusConfig() (string, error) {
-	// find master file from MultusAutoconfigDir
+func (o *Options) getMasterConfigPath() (string, error) {
+	// Master config file is specified
+	if o.MultusMasterCNIFileName != "" {
+		return filepath.Join(o.MultusAutoconfigDir, o.MultusMasterCNIFileName), nil
+	}
+
+	// Pick the alphabetically first config file from MultusAutoconfigDir
 	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
 	if err != nil {
 		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
 	}
 
-	masterConfigPath := files[0]
-	masterConfigBytes, err := os.ReadFile(masterConfigPath)
-	if err != nil {
-		return "", fmt.Errorf("cannot read master CNI config file %q: %v", masterConfigPath, err)
+	for _, filename := range files {
+		if !strings.HasPrefix(filepath.Base(filename), "00-multus.conf") {
+			return filename, nil
+		}
 	}
+
+	// No config file found
+	return "", fmt.Errorf("cannot find valid master CNI config in %q", o.MultusAutoconfigDir)
+}
+
+func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, []byte, error) {
+	masterConfigPath, err := o.getMasterConfigPath()
+	if err != nil {
+		return "", nil, err
+	}
+
+	masterConfigBytes, masterConfigFileHash, err := getFileAndHash(masterConfigPath)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if prevMasterConfigFileHash != nil && bytes.Equal(prevMasterConfigFileHash, masterConfigFileHash) {
+		return masterConfigPath, masterConfigFileHash, nil
+	}
+
+	if prevMasterConfigFileHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("master config changed - recreating multus config\n")
+	}
+
 	masterConfig := map[string]interface{}{}
 	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
-		return "", fmt.Errorf("cannot read master CNI config json: %v", err)
+		return "", nil, fmt.Errorf("cannot read master CNI config json: %v", err)
 	}
 
 	// check CNIVersion
 	masterCNIVersionElem, ok := masterConfig["cniVersion"]
 	if !ok {
-		return "", fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
+		return "", nil, fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
 	}
 
 	if o.ForceCNIVersion {
@@ -316,7 +374,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	} else {
 		masterCNIVersion := masterCNIVersionElem.(string)
 		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
-			return "", fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
+			return "", nil, fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
 		}
 		o.CNIVersion = masterCNIVersion
 	}
@@ -327,7 +385,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if o.OverrideNetworkName {
 		masterPluginNetworkElem, ok := masterConfig["name"]
 		if !ok {
-			return "", fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
+			return "", nil, fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
 		}
 
 		masterPluginNetworkName = masterPluginNetworkElem.(string)
@@ -341,7 +399,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if isMasterConfList {
 		masterPluginsElem, ok := masterConfig["plugins"]
 		if !ok {
-			return "", fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
+			return "", nil, fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
 		}
 		masterPlugins := masterPluginsElem.([]interface{})
 		for _, v := range masterPlugins {
@@ -368,7 +426,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	if len(masterCapabilities) != 0 {
 		capabilitiesByte, err := json.Marshal(masterCapabilities)
 		if err != nil {
-			return "", fmt.Errorf("cannot get capabilities map: %v", err)
+			return "", nil, fmt.Errorf("cannot get capabilities map: %v", err)
 		}
 		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
 	}
@@ -400,7 +458,7 @@ func (o *Options) createMultusConfig() (string, error) {
 	case "":
 		// no logLevel config, skipped
 	default:
-		return "", fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
+		return "", nil, fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
 	}
 
 	// check MultusLogFile
@@ -415,6 +473,12 @@ func (o *Options) createMultusConfig() (string, error) {
 		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
 	}
 
+	// check MultusCNIConfDir
+	multusCNIConfDirConfig := ""
+	if o.MultusCNIConfDir != "" {
+		multusCNIConfDirConfig = fmt.Sprintf("\n        \"cniConf\": %q,", o.MultusCNIConfDir)
+	}
+
 	// check ReadinessIndicatorFile
 	readinessIndicatorFileConfig := ""
 	if o.ReadinessIndicatorFile != "" {
@@ -424,28 +488,28 @@ func (o *Options) createMultusConfig() (string, error) {
 	// fill .MasterPluginJSON
 	masterPluginByte, err := json.Marshal(masterConfig)
 	if err != nil {
-		return "", fmt.Errorf("cannot encode master CNI config: %v", err)
+		return "", nil, fmt.Errorf("cannot encode master CNI config: %v", err)
 	}
 
 	// generate multus config
 	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
 	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
 	if err != nil {
-		return "", fmt.Errorf("cannot create multus cni temp file: %v", err)
+		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
 	}
 
-	// use conflist template if cniVersionConfig == "1.0.0"
+	// use conflist template if cniVersionConfig >= "1.0.0"
 	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
 	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
 	if err != nil {
-		return "", fmt.Errorf("template parse error: %v", err)
+		return "", nil, fmt.Errorf("template parse error: %v", err)
 	}
 
-	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
+	if gt, err := cniversion.GreaterThanOrEqualTo(o.CNIVersion, "1.0.0"); err == nil && gt {
 		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
 		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
 		if err != nil {
-			return "", fmt.Errorf("template parse error: %v", err)
+			return "", nil, fmt.Errorf("template parse error: %v", err)
 		}
 	}
 
@@ -459,37 +523,38 @@ func (o *Options) createMultusConfig() (string, error) {
 		"LogLevelConfig":               logLevelConfig,
 		"LogFileConfig":                logFileConfig,
 		"AdditionalBinDirConfig":       additionalBinDirConfig,
+		"MultusCNIConfDirConfig":       multusCNIConfDirConfig,
 		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
 		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
 		"MasterPluginJSON":             string(masterPluginByte),
 	}
 	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
-		return "", fmt.Errorf("cannot create multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot create multus cni config: %v", err)
 	}
 
 	if err := fp.Sync(); err != nil {
 		os.Remove(tempFileName)
-		return "", fmt.Errorf("cannot flush multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot flush multus cni config: %v", err)
 	}
 	if err := fp.Close(); err != nil {
 		os.Remove(tempFileName)
-		return "", fmt.Errorf("cannot close multus cni config: %v", err)
+		return "", nil, fmt.Errorf("cannot close multus cni config: %v", err)
 	}
 
 	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
-		return "", fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
+		return "", nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
 	}
 
 	if o.RenameConfFile {
 		//masterConfigPath
 		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
 		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
-			return "", fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
+			return "", nil, fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
 		}
 		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
 	}
 
-	return masterConfigPath, nil
+	return masterConfigPath, masterConfigFileHash, nil
 }
 
 func main() {
@@ -518,10 +583,15 @@ func main() {
 		}
 	}
 
-	var kubeConfigHash []byte
+	var masterConfigHash, caHash, saTokenHash []byte
 	var masterConfigFilePath string
 	// copy user specified multus conf to CNI conf directory
 	if opt.MultusConfFile != "auto" {
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
+			return
+		}
 		confFileName := filepath.Base(opt.MultusConfFile)
 		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
 		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
@@ -530,13 +600,13 @@ func main() {
 		}
 		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
 	} else { // auto generate multus config
-		kubeConfigHash, err = opt.createKubeConfig(nil)
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
 			return
 		}
 		fmt.Printf("kubeconfig file is created.\n")
-		masterConfigFilePath, err = opt.createMultusConfig()
+		masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(nil)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
 			return
@@ -544,42 +614,72 @@ func main() {
 		fmt.Printf("multus config file is created.\n")
 	}
 
-	if opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" {
+	ctx := signals.SetupSignalHandler()
+
+	if opt.CleanupConfigOnExit {
+		defer cleanupMultusConf(&opt)
+	}
+
+	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
+	if watchChanges {
 		fmt.Printf("Entering watch loop...\n")
-		for {
-			// Check kubeconfig and update if different (i.e. service account updated)
-			kubeConfigHash, err = opt.createKubeConfig(kubeConfigHash)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
-				return
-			}
+		masterConfigExists := true
 
-			// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
-			_, err = os.Stat(masterConfigFilePath)
-
-			// if masterConfigFilePath is no longer exists
-			if os.IsNotExist(err) {
-				fmt.Printf("Master plugin @ %q has been deleted. Allowing 45 seconds for its restoration...\n", masterConfigFilePath)
-				time.Sleep(10 * time.Second)
-
-				for range time.Tick(1 * time.Second) {
-					_, err = os.Stat(masterConfigFilePath)
-					if !os.IsNotExist(err) {
-						fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
-						break
-					}
+	outer:
+		for range time.Tick(1 * time.Second) {
+			select {
+			case <-ctx.Done():
+				// signal received break from loop
+				break outer
+			default:
+				// Check kubeconfig and update if different (i.e. service account updated)
+				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
+					return
+				}
+
+				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
+				_, err = os.Stat(masterConfigFilePath)
+
+				// if masterConfigFilePath is no longer exists
+				if os.IsNotExist(err) {
+					if masterConfigExists {
+						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
+					}
+					masterConfigExists = false
+					continue
+				}
+
+				if !masterConfigExists {
+					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
+					masterConfigExists = true
+				}
+
+				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+					return
 				}
-			}
-			masterConfigFilePath, err = opt.createMultusConfig()
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
-				return
 			}
 		}
 	} else {
-		// sleep infinitely
-		for {
-			time.Sleep(time.Duration(1<<63 - 1))
-		}
+		// wait until signal received
+		<-ctx.Done()
 	}
 }
+
+func cleanupMultusConf(opt *Options) {
+	// try remove multus conf
+	if opt.MultusConfFile == "auto" {
+		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+	} else {
+		confFileName := filepath.Base(opt.MultusConfFile)
+		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
+	}
+
+}

cmd/thin_entrypoint/main_test.go

@@ -1,14 +1,39 @@
 package main
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"syscall"
 	"testing"
 
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo/v2" //nolint:golint
+	. "github.com/onsi/gomega"    //nolint:golint
 )
 
+// chrootTestHelper performs chroot syscall, returns func to get back to original root or error if occurred
+func chrootTestHelper(path string) (func() error, error) {
+	root, err := os.Open("/")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := syscall.Chroot(path); err != nil {
+		root.Close()
+		return nil, err
+	}
+
+	return func() error {
+		defer root.Close()
+		if err := root.Chdir(); err != nil {
+			return err
+		}
+		return syscall.Chroot(".")
+	}, nil
+}
+
 func TestThinEntrypoint(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "thin_entrypoint")
@@ -103,14 +128,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -151,14 +177,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -200,7 +227,7 @@ var _ = Describe("thin entrypoint testing", func() {
 		err = os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)
 		Expect(err).NotTo(HaveOccurred())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
@@ -210,11 +237,13 @@ var _ = Describe("thin entrypoint testing", func() {
 			MultusLogLevel:           "DEBUG",
 			MultusLogFile:            "/tmp/foobar.log",
 			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
 			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult := `{
         "cniVersion": "0.3.1",
@@ -225,6 +254,7 @@ var _ = Describe("thin entrypoint testing", func() {
         "logLevel": "debug",
         "logFile": "/tmp/foobar.log",
         "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
         "readinessindicatorfile": "/var/lib/foobar_indicator",
         "kubeconfig": "/etc/foobar_kubeconfig",
         "delegates": [
@@ -258,14 +288,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -287,6 +318,56 @@ var _ = Describe("thin entrypoint testing", func() {
 		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
+	It("Run createMultusConfig(), default, conflist for cniVersion 1.1.0", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.1.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.1.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.1.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
 	It("Run createMultusConfig(), capabilities, conflist", func() {
 		// create directory and files
 		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
@@ -308,14 +389,15 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -358,7 +440,7 @@ var _ = Describe("thin entrypoint testing", func() {
 		}`
 		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
 
-		masterConfigPath, err := (&Options{
+		masterConfigPath, masterConfigHash, err := (&Options{
 			MultusAutoconfigDir:      multusAutoConfigDir,
 			CNIConfDir:               cniConfDir,
 			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
@@ -368,11 +450,13 @@ var _ = Describe("thin entrypoint testing", func() {
 			MultusLogLevel:           "DEBUG",
 			MultusLogFile:            "/tmp/foobar.log",
 			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
 			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
-		}).createMultusConfig()
+		}).createMultusConfig(nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
 
 		expectedResult :=
 			`{
@@ -385,6 +469,7 @@ var _ = Describe("thin entrypoint testing", func() {
         "logLevel": "debug",
         "logFile": "/tmp/foobar.log",
         "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
         "readinessindicatorfile": "/var/lib/foobar_indicator",
         "kubeconfig": "/etc/foobar_kubeconfig",
         "delegates": [
@@ -399,4 +484,111 @@ var _ = Describe("thin entrypoint testing", func() {
 		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
+	It("Run createMultusConfig(), with options, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfigFileName := "10-testcni.conf"
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, masterCNIConfigFileName), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		// create another CNI config
+		anotherCNIConfigFileName := "09-test2cni.conf" // Alphabetically before masterCNIConfigFileName
+		anotherCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test2",
+			"type": "cnitest2type"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, anotherCNIConfigFileName), []byte(anotherCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			MultusMasterCNIFileName:  masterCNIConfigFileName,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createKubeConfig()", func() {
+		// create temp dir and files
+		tmpDir := GinkgoT().TempDir()
+
+		cniConfDir := "/cni_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, cniConfDir), 0755)).To(Succeed())
+
+		multusConfDir := "/multus_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, multusConfDir), 0755)).To(Succeed())
+
+		// Create service account CA file and token file with dummy data
+		svcAccountPath := filepath.Join(tmpDir, "var/run/secrets/kubernetes.io/serviceaccount")
+		Expect(os.MkdirAll(svcAccountPath, 0755)).ToNot(HaveOccurred())
+		svcAccountCAFile := filepath.Join(tmpDir, serviceAccountCAFile)
+		svcAccountTokenFile := filepath.Join(tmpDir, serviceAccountTokenFile)
+		Expect(os.WriteFile(svcAccountCAFile, []byte("dummy-ca-content"), 0644)).To(Succeed())
+		Expect(os.WriteFile(svcAccountTokenFile, []byte("dummy-token-content"), 0644)).To(Succeed())
+
+		// Set up the Options struct
+		options := &Options{
+			CNIConfDir:       cniConfDir,
+			MultusCNIConfDir: multusConfDir,
+		}
+
+		// Run the createKubeConfig function in a chroot env
+		back, err := chrootTestHelper(tmpDir)
+		Expect(err).ToNot(HaveOccurred())
+		caHash, saTokenHash, err := options.createKubeConfig(nil, nil)
+		Expect(back()).ToNot(HaveOccurred())
+		// back to original root
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(caHash).NotTo(BeNil())
+		Expect(saTokenHash).NotTo(BeNil())
+
+		// Verify the kubeconfig file was created successfully
+		kubeConfigPath := filepath.Join(tmpDir, cniConfDir, "multus.d", "multus.kubeconfig")
+		content, err := os.ReadFile(kubeConfigPath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(content).NotTo(BeEmpty())
+
+		// Cleanup
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
 })

deployments/multus-daemonset-crio.yml

@@ -197,6 +197,7 @@ spec:
           privileged: true
           capabilities:
             add: ["SYS_ADMIN"]
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: run
           mountPath: /run

deployments/multus-daemonset-thick.yml

@@ -69,7 +69,9 @@ rules:
       - pods/status
     verbs:
       - get
+      - list
       - update
+      - watch
   - apiGroups:
       - ""
       - events.k8s.io
@@ -111,13 +113,13 @@ data:
   daemon-config.json: |
     {
         "chrootDir": "/hostroot",
-        "confDir": "/host/etc/cni/net.d",
-        "logLevel": "verbose",
-        "socketDir": "/host/run/multus/",
         "cniVersion": "0.3.1",
+        "logLevel": "verbose",
+        "logToStderr": true,
         "cniConfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
         "multusConfigFile": "auto",
-        "multusAutoconfigDir": "/host/etc/cni/net.d"
+        "socketDir": "/host/run/multus/"
     }
 ---
 apiVersion: apps/v1
@@ -163,15 +165,22 @@ spec:
               memory: "50Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
+            # multus-daemon expects that cnibin path must be identical between pod and container host.
+            # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
+            # not to any other directory, like '/opt/bin' or '/usr/bin'.
+            - name: cnibin
+              mountPath: /opt/cni/bin
             - name: host-run
               mountPath: /host/run
             - name: host-var-lib-cni-multus
               mountPath: /var/lib/cni/multus
             - name: host-var-lib-kubelet
               mountPath: /var/lib/kubelet
+              mountPropagation: HostToContainer
             - name: host-run-k8s-cni-cncf-io
               mountPath: /run/k8s.cni.cncf.io
             - name: host-run-netns
@@ -183,19 +192,29 @@ spec:
             - name: hostroot
               mountPath: /hostroot
               mountPropagation: HostToContainer
+            - mountPath: /etc/cni/multus/net.d
+              name: multus-conf-dir
+          env:
+            - name: MULTUS_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
       initContainers:
         - name: install-multus-binary
           image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
           command:
-            - "cp"
-            - "/usr/src/multus-cni/bin/multus-shim"
-            - "/host/opt/cni/bin/multus-shim"
+            - "/usr/src/multus-cni/bin/install_multus"
+            - "-d"
+            - "/host/opt/cni/bin"
+            - "-t"
+            - "thick"
           resources:
             requests:
               cpu: "10m"
               memory: "15Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin
@@ -232,3 +251,6 @@ spec:
         - name: host-run-netns
           hostPath:
             path: /run/netns/
+        - name: multus-conf-dir
+          hostPath:
+            path: /etc/cni/multus/net.d

deployments/multus-daemonset.yml

@@ -194,6 +194,7 @@ spec:
             memory: "50Mi"
         securityContext:
           privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: cni
           mountPath: /host/etc/cni/net.d
@@ -214,6 +215,7 @@ spec:
               memory: "15Mi"
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin

docs/configuration.md

@@ -1,7 +1,16 @@
-## Multus-cni Configuration Reference
+# Multus-cni Configuration Reference
+
+## Introduction
+
+Aside from setting options for Multus, one of the goals of configuration is to set the configuration for your *default network*. The default network is also sometimes referred as the "primary CNI plugin", the "primary network", or a "default CNI plugin" and is the CNI plugin that is used to implement [the Kubernetes networking model](https://kubernetes.io/docs/concepts/services-networking/#the-kubernetes-network-model) in your cluster. Common examples include Flannel, Weave, Calico, Cilium, and OVN-Kubernetes, among others.
+
+Here we will refer to this as your default CNI plugin or default network.
+
+## Example configuration
 
 Following is the example of multus config file, in `/etc/cni/net.d/`.
-(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
+
+Example configuration using `clusterNetwork` (see also [using delegates](#using-delegates))
 
 ```
 {
@@ -23,65 +32,109 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     "capabilities": {
         "portMappings": true
     },    
-    "readinessindicatorfile": "",
     "namespaceIsolation": false,
-"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
-    "clusterNetwork": "defaultCRD",
-    "defaultNetworks": ["sidecarCRD", "flannel"],
+    "clusterNetwork": "/etc/cni/net.d/99-flannel.conf",
+    "defaultNetworks": ["sidecarCRD", "exampleNetwork"],
     "systemNamespaces": ["kube-system", "admin"],
     "multusNamespace": "kube-system",
-"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
+    "auxiliaryCNIChainName": "cni-chain-config",
+    allowTryDeleteOnErr: false
+}
+```
+
+## Index of configuration options
+
+This is a general index of options, however note that you must set either the `clusterNetwork` or `delegates` options, see the following sections after the index for details.
+
+* `name` (string, required): The name of the network
+* `type` (string, required): Must be set to the value of "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* [`logToStderr`](#Logging-via-STDERR) (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
+* [`logFile`](#Writing-to-a-Log-File) (string, optional): file path for log file. multus puts log in given file
+* [`logLevel`](#Logging-Level) (string, optional): logging level (values in decreasing order of verbosity: "debug", "error", "verbose", or "panic")
+* [`logOptions`](#Logging-Options) (object, optional): logging option, More detailed log configuration
+* [`namespaceIsolation`](#Namespace-Isolation) (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* [`globalNamespaces`](#Allow-specific-namespaces-to-be-used-across-namespaces-when-using-namespace-isolation): (string, optional): Used only when `namespaceIsolation` is true, allows specification of comma-delimited list of namespaces which may be referred to outside of namespace isolation.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* [`readinessindicatorfile`](#Default-Network-Readiness-Indicator): The path to a file whose existence denotes that the default network is ready
+message to next when some missing error. Defaults to false.
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` (the default value is `kube-system`)
+* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL 
+* [`auxiliaryCNIChainName`](#auxiliaryCNIChainName) (string, optional): Enable loading CNI configurations from disk as chained plugins in an auxiliary CNI chain
+
+### Using `clusterNetwork`
+
+Using the `clusterNetwork` option and the `delegates` are **mutually exclusive**. If `clusterNetwork` is set, the `delegates` field is *ignored*. 
+
+You **must** set one or the other.
+
+Therefore:
+
+* Set `clusterNetwork` and if this is set, optionally set the `defaultNetworks`.
+* OR you **must** set `delegates`.
+
+Options:
+
+* `clusterNetwork` (string, required if not using `delegates`): the default CNI plugin to be executed.
+* `defaultNetworks` ([]string, optional): Additional / secondary network attachment that is always attached to each pod. 
+
+The following values are valid for both `clusterNetwork` and `defaultNetworks` and are processed in the following order:
+
+* The name of a `NetworkAttachmentDefinition` custom resource in the namespace specified  by the `multusNamespace` configuration option
+* The `"name"` value in the contents of a CNI JSON configuration file in the CNI configuration directory, 
+  * The given name for `clusterNetwork` should match the value for `name` key in the contents of the CNI JSON file (e.g. `"name": "test"` in `my.conf` when `"clusterNetwork": "test"`)
+* A path to a directory containing CNI json configuration files. The alphabetically first file will be used.
+* Absolute file path for CNI config file
+* If none of the above are found using the value, Multus will raise an error.
+
+If for example you have `defaultNetworks` set as:
+
+```
+"defaultNetworks": ["sidecarNetwork", "exampleNetwork"],
+```
+
+In this example, the values in the expression refer to `NetworkAttachmentDefinition` custom resource names. Therefore, there must be `NetworkAttachmentDefinitions` already created with the names `sidecarNetwork` and `exampleNetwork`.
+
+This means that in addition to the cluster network, each pod would be assigned two additional networks by default, and the pod would present three interfaces, e.g. `eth0`, `net1`, and `net2`, with `net1` and `net2` being set by the above described `NetworkAttachmentDefinitions`. Additional attachments as made by setting `k8s.v1.cni.cncf.io/networks` on pods will be made in addition to those set in the `defaultNetworks` configuration option.
+
+### Using `delegates`
+
+If `clusterNetwork` is not set, you **must** use `delegates`.
+
+* `delegates` ([]map, required if not using `clusterNetwork`). List of CNI configurations to be used as your default CNI plugin(s).
+
+Example configuration using `delegates`:
+
+```
+{
+    "cniVersion": "0.3.1",
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
     "delegates": [{
         "type": "weave-net",
         "hairpinMode": true
     }, {
         "type": "macvlan",
         ... (snip)
-    }],
-    allowTryDeleteOnErr: false
+    }]
 }
 ```
 
-* `name` (string, required): the name of the network
-* `type` (string, required): "multus"
-* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
-* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
-* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
-* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
-* `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
-* `logFile` (string, optional): file path for log file. multus puts log in given file
-* `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
-* `logOptions` (object, optional): logging option, More detailed log configuration
-* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
-* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
-* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
-
-User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
-
-* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
-* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist), directory for CNI config file or absolute file path for CNI config file
-* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
-* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
-* `delegates` ([]map,required): number of delegate details in the Multus
-* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL message to next when some missing error. Defaults to false.
-
-### Network selection flow of clusterNetwork/defaultNetworks
-
-Multus will find network for clusterNetwork/defaultNetworks as following sequences:
-
-1. CRD object for given network name, in 'kube-system' namespace
-1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
-1. Directory for CNI json config file. Multus will find alphabetically first file for the network
-1. File path for CNI json confile file.
-1. Multus failed to find network. Multus raise error message
-
-## Miscellaneous config
+## Configuration Option Details
 
 ### Default Network Readiness Indicator
 
-You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
+You may desire that your default network becomes ready before attaching networks with Multus. This is disabled by default and not used unless you set the `readinessindicatorfile` option to a non-blank value.
 
-For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, therefore, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
 
 In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
 
@@ -329,3 +382,47 @@ annotations:
  v1.multus-cni.io/default-network: calico-conf
 ...
 ```
+
+### `auxiliaryCNIChainName`
+
+`auxiliaryCNIChainName` (of value string) is used to express the name of an additional auxiliary CNI chain that will execute in order to composably execute chained CNI plugins from configurations on the host's disk in a subdirectory of the CNI configuration directory.
+
+**NOTE**: The path used to determine the base for the subdirectory is the pathname of the `clusterNetwork` value, which must be set to a file in order to use this functionality.
+
+When this string is set, Multus will execute an additional CNI chain, outside of the default network, on its own independent CNI chain (as to not interfere with default network functionality that might be hampered by CNI chaining and to otherwise isolate this execution) and will load CNI configurations from a subdirectory of the same name in the CNI configuration directory.
+
+This feature is based on [improvements made to libcni for "safe subdirectory-based plugin conf loading"](https://github.com/containernetworking/cni/pull/1052). 
+
+`auxiliaryCNIChainName` is meant to be set as a CNI configuration name, this name is arbitrary but must match the subdirectory name.
+
+Consider this [daemon configuration](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/deployments/multus-daemonset-thick.yml#L113):
+
+```
+{
+  "cniConfigDir": "/host/etc/cni/net.d",
+  "multusAutoconfigDir": "/host/etc/cni/net.d",
+  "multusConfigFile": "auto",
+  "socketDir": "/host/run/multus/",
+  "auxiliaryCNIChainName": "cni-chain-config"
+}
+```
+
+Here we have set `"auxiliaryCNIChainName": "cni-chain-config"`, and we have expressed that our CNI configurations are on `/etc/cni/net.d/` on the host.
+
+In this case, we would also have a directory named in `/etc/cni/net.d/cni-chain-config`
+
+One could add any number of CNI configurations to be used as part of this chain, consider this example if we added a tuning CNI configuration called `/etc/cni/net.d/cni-chain-config/mytuning.conf` with these contents:
+
+```
+{
+  "name": "mytuning",
+  "type": "tuning",
+  "sysctl": {
+    "net.ipv4.conf.IFNAME.arp_filter": "1"
+  }
+}
+```
+
+With the given configuration, plus this configuration, this would be executed for every pod launched by Multus CNI.
+
+If this is unset, no auxiliary chain will be executed. However, if the default network CNI configuration is loaded from disk and is a conflist format, the libcni functionality for loading from a subdirectory will still apply.

docs/development.md

@@ -39,7 +39,7 @@ cd multus-cni
 ./hack/build-go.sh
 ```
 
-## How do I run CI tests?
+## How do I run the unit tests?
 
 Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
 
@@ -47,6 +47,10 @@ Multus has go unit tests (based on ginkgo framework).The following commands driv
 sudo ./hack/test-go.sh
 ```
 
+## How do I run the e2e tests?
+
+Check the `README.md` in the `./e2e/` folder.
+
 ## What are the best practices for logging?
 
 The following are the best practices for multus logging:
@@ -59,3 +63,7 @@ The following are the best practices for multus logging:
 ## Multus release schedule
 
 On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.
+
+## Multi-arch builds
+
+Multus is currently built for a number of architectures, however, our testing and validation is only performed against x86 architectures. Our x86 architecture has end to end testing, however, for other architectures, only supported via best effort community contributions.

docs/how-to-use.md

@@ -19,13 +19,13 @@ You may acquire the Multus binary via compilation (see the [developer guide](dev
 
 *Via Daemonset method*
 
-As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
+As a [quickstart](quickstart.md), you may apply these YAML files. Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
 
-    cat ./deployments/multus-daemonset.yml | kubectl apply -f -  # thin deployment
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml  # thin deployment
 
 or
 
-    cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f - # thick (client/server) deployment
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml # thick (client/server) deployment
 
 If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
 
@@ -126,7 +126,7 @@ Create kubeconfig at master node as following commands:
 mkdir -p /etc/cni/net.d/multus.d
 SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
 SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_PROTOCOL=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
 KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
 KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
 cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
@@ -511,7 +511,7 @@ spec:
 EOF
 ```
 
-We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation. 
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation.
 
 ```
 cat <<EOF | kubectl create -f -
@@ -537,9 +537,9 @@ This will set `192.168.2.1` as the default route over the `net1` interface, such
 ```
 kubectl exec -it samplepod -- ip route
 
-default via 192.168.2.1 dev net1 
-10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
-10.244.0.0/16 via 10.244.0.1 dev eth0 
+default via 192.168.2.1 dev net1
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169
+10.244.0.0/16 via 10.244.0.1 dev eth0
 ```
 
 ## Entrypoint Parameters
@@ -551,7 +551,7 @@ Typically, you'd modified the daemonset YAML itself to specify these parameters.
 For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
 
 ```
-  command: ["/entrypoint.sh"]
+  command: ["/thin_entrypoint"]
   args:
   - "--multus-conf-file=auto"
   - "--namespace-isolation=true"
@@ -590,7 +590,7 @@ The `--multus-conf-file` is one of two options; it can be set to a source file t
 
 The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
 
-In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will take the alphabetically first configuration there and wrap that into a Multus configuration.
 
     --multus-autoconfig-dir=/host/etc/cni/net.d
 
@@ -619,6 +619,14 @@ In some cases, the original CNI configuration that the Multus configuration was
 
     --cleanup-config-on-exit=true
 
+When specifying `--cleanup-config-on-exit=true` the entrypoint script will delete any generated/copied Multus configuration files when entrypoint script
+exits (upon Pod termination). This allows Multus to be safely removed from the cluster when its no longer needed.
+
+In addition, when both `--cleanup-config-on-exit=true` and `--multus-conf-file=auto` are specified, the entrypoint script will watch for changes of the
+master CNI configuration and kubeconfig. when such change detected, the script will re-genereate Multus configuration. Watch can be skipped by setting:
+
+    --skip-config-watch
+
 Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
 
     --rename-conf-file=true
@@ -634,3 +642,126 @@ Sometimes, you may wish to not have the entrypoint copy the binary file onto the
 If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
 
     --readiness-indicator-file=/path/to/file
+
+### Run pod with network annotation and Dynamic Resource Allocation driver
+
+> :warning: Dynamic Resource Allocation (DRA) is [currently an alpha](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/),
+> and is subject to change. Please consider this functionality as a preview. The architecture and usage of DRA in
+> Multus CNI may change in the future as this technology matures.
+>
+> The current DRA integration is based on the DRA API for Kubernetes 1.26 to 1.30. With Kubernetes 1.31, the DRA API
+> will change and multus doesn't integrate with the new API yet.
+
+Dynamic Resource Allocation is alternative mechanism to device plugin which allows to requests pod and container
+resources.
+
+The following sections describe how to use DRA with multus and NVIDIA DRA driver. Other DRA networking driver vendors
+should follow similar concepts to make use of multus DRA support.
+
+#### Prerequisite
+
+1. Kubernetes 1.27
+2. Container Runtime with CDI support enabled
+3. Kubernetes runtime-config=resource.k8s.io/v1alpha2
+4. Kubernetes feature-gates=DynamicResourceAllocation=True,KubeletPodResourcesDynamicResources=true
+
+#### Install DRA driver
+
+The current example uses NVIDIA DRA driver for networking. This DRA driver is not publicly available. An alternative to
+this DRA driver is available at [dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver).
+
+#### Create dynamic resource class with NVIDIA network DRA driver
+
+The `ResourceClass` defines the resource pool of `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClass
+metadata:
+  name: sf-pool-1
+driverName: net.resource.nvidia.com
+EOF
+```
+
+#### Create network attachment definition with resource name
+
+The `k8s.v1.cni.cncf.io/resourceName` should match the `ResourceClass` name defined in the section above.
+In this example it is `sf-pool-1`. Multus query the K8s PodResource API to fetch the `resourceClass` name and also
+query the NetworkAttachmentDefinition `k8s.v1.cni.cncf.io/resourceName`. If both has the same name multus send the
+CDI device name in the DeviceID argument.
+
+##### NetworkAttachmentDefinition for ovn-kubernetes example:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: sf-pool-1
+spec:
+  config: '{
+      "cniVersion": "0.4.0",
+      "dns": {},
+      "ipam": {},
+      "logFile": "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+      "logLevel": "4",
+      "logfile-maxage": 5,
+      "logfile-maxbackups": 5,
+      "logfile-maxsize": 100,
+      "name": "ovn-kubernetes",
+      "type": "ovn-k8s-cni-overlay"
+    }'
+EOF
+```
+
+#### Create DRA Resource Claim
+
+Following command creates `ResourceClaim` `sf` which request resource from  `ResourceClass` `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaim
+metadata:
+  namespace: default
+  name: sf
+spec:
+  spec:
+    resourceClassName: sf-pool-1
+EOF
+```
+
+#### Launch pod with DRA Resource Claim
+
+Following command Launch a Pod with primiry network `default` and `ResourceClaim` `sf`.
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: default
+  name: test-sf-claim
+  annotations:
+    v1.multus-cni.io/default-network: default
+spec:
+  restartPolicy: Always
+  containers:
+  - name: with-resource
+    image: docker.io/library/ubuntu:22.04
+    command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+    resources:
+      claims:
+      - name: resource
+  resourceClaims:
+  - name: resource
+    source:
+      resourceClaimName: sf
+```

docs/quickstart.md

@@ -42,25 +42,19 @@ master-2              Ready    master          1h     v1.17.1
 
 Our recommended quickstart method to deploy Multus is to deploy using a Daemonset (a method of running pods on each nodes in your cluster), this spins up pods which install a Multus binary and configure Multus for usage.
 
-Firstly, clone this GitHub repository. 
-
-```
-git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
-```
-
 We'll apply a YAML file with `kubectl` from this repo, which installs the Multus components.
 
 Recommended installation:
 
 ```
-cat ./deployments/multus-daemonset-thick.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
 See the [thick plugin docs](./thick-plugin.md) for more information about this architecture.
 
 Alternatively, you may install the thin-plugin with:
 
 ```
-cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 ```
 
 ### What the Multus daemonset does

docs/thick-plugin.md

@@ -72,6 +72,7 @@ is provided.
 - `"logLevel"`: the logging level for the multus daemon logs.
 - `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
 as well. By default, it is disabled.
+- `"auxiliaryCNIChainName"`: set a value to execute chained cni configurations from disk in an auxiliary CNI chain (see details in [configuration.md](configuration.md))
 
 In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
 

e2e/README.md

@@ -5,7 +5,7 @@
 To run the e2e test, you need the following components:
 
 - curl
-- j2cli
+- jinjanator (optional)
 - docker
 
 ### How to test e2e
@@ -14,7 +14,23 @@ To run the e2e test, you need the following components:
 $ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
 $ cd multus-cni/e2e
 $ ./get_tools.sh
+```
+
+If you have `jinjanator` you can generate the YAML with:
+
+```
 $ ./generate_yamls.sh
+```
+
+Alternatively, if you have trouble with it, use the `sed` script.
+
+```
+$ ./e2e/sed_generate_yaml.sh
+```
+
+Then, setup the cluster
+
+```
 $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```

e2e/get_tools.sh

@@ -5,7 +5,7 @@ if [ ! -d bin ]; then
 	mkdir bin
 fi
 
-curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.27.0/kind-$(uname)-amd64"
 chmod +x ./bin/kind
 curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 chmod +x ./bin/kubectl
@@ -13,3 +13,4 @@ curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq
+wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm

e2e/sed_generate_yaml.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ ! -d yamls ]; then
+	mkdir yamls
+fi
+
+# specify CNI version (default: 0.4.0)
+CNI_VERSION=${CNI_VERSION:-0.4.0}
+
+templates_dir="$(dirname $(readlink -f $0))/templates"
+
+# generate yaml files based on templates/*.j2 to yamls directory
+for i in `ls ${templates_dir}/*.j2`; do
+	echo "Processing $i..."
+	# Use sed to replace the placeholder with the CNI_VERSION variable
+	sed "s/{{ CNI_VERSION }}/$CNI_VERSION/g" $i > yamls/$(basename ${i%.j2})
+done

e2e/setup_cluster.sh

@@ -11,33 +11,39 @@ OCI_BIN="${OCI_BIN:-docker}"
 # Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
 # Defaults to `multus-daemonset-thick.yml`.
 MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
+# define the dockerfile to build multus.
+# Acceptable values are `Dockerfile`. `Dockerfile.thick`.
+# Defaults to `Dockerfile.thick`.
+MULTUS_DOCKERFILE="${MULTUS_DOCKERFILE:-Dockerfile.thick}"
 
 kind_network='kind'
-reg_name='kind-registry'
-reg_port='5000'
-running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
-if [ "${running}" != 'true' ]; then
-  # run registry and push the multus image
-  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
-  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
-  $OCI_BIN push localhost:5000/multus:e2e
+if [ "${MULTUS_DOCKERFILE}" != "none" ]; then
+	$OCI_BIN build -t localhost:5000/multus:e2e -f ../images/${MULTUS_DOCKERFILE} ..
 fi
-reg_host="${reg_name}"
-if [ "${kind_network}" = "bridge" ]; then
-    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
-fi
-echo "Registry Host: ${reg_host}"
 
 # deploy cluster with kind
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-containerdConfigPatches:
-- |-
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_host}:${reg_port}"]
 nodes:
   - role: control-plane
+    kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          runtime-config: "resource.k8s.io/v1beta1=true"
+      scheduler:
+        extraArgs:
+          v: "1"
+      controllerManager:
+        extraArgs:
+          v: "1"
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          v: "1"
   - role: worker
     kubeadmConfigPatches:
     - |
@@ -45,31 +51,44 @@ nodes:
       nodeRegistration:
         kubeletExtraArgs:
           pod-manifest-path: "/etc/kubernetes/manifests/"
+          feature-gates: "DynamicResourceAllocation=true,DRAResourceClaimDeviceStatus=true,KubeletPodResourcesDynamicResources=true"
+    - |
+      kind: JoinConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          v: "1"
   - role: worker
+    kubeadmConfigPatches:
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          pod-manifest-path: "/etc/kubernetes/manifests/"
+          feature-gates: "DynamicResourceAllocation=true,DRAResourceClaimDeviceStatus=true,KubeletPodResourcesDynamicResources=true"
+    - |
+      kind: JoinConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          v: "1"
+# Required by DRA Integration
+##
+featureGates:
+  DynamicResourceAllocation: true
+  DRAResourceClaimDeviceStatus: true
+  KubeletPodResourcesDynamicResources: true
+runtimeConfig:
+  "api/beta": "true"
+containerdConfigPatches:
+# Enable CDI as described in
+# https://github.com/container-orchestrated-devices/container-device-interface#containerd-configuration
+- |-
+  [plugins."io.containerd.grpc.v1.cri"]
+      enable_cdi = true
+##
 EOF
 
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: local-registry-hosting
-  namespace: kube-public
-data:
-  localRegistryHosting.v1: |
-    host: "localhost:${reg_port}"
-    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
-EOF
-
-containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
-needs_connect="true"
-for c in $containers; do
-  if [ "$c" = "${reg_name}" ]; then
-    needs_connect="false"
-  fi
-done
-if [ "${needs_connect}" = "true" ]; then
-  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
-fi
+# load multus image from container host to kind node
+kind load docker-image localhost:5000/multus:e2e
 
 worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
 worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
@@ -83,4 +102,4 @@ sleep 1
 kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
 kubectl create -f yamls/cni-install.yml
 sleep 1
-kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s
+kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=400s

e2e/teardown.sh

@@ -1,10 +1,7 @@
 #!/bin/sh
 #set -o errexit
 
-reg_name='kind-registry'
 export PATH=${PATH}:./bin
 
 # delete cluster kind
 kind delete cluster
-docker kill ${reg_name}
-docker rm ${reg_name}

e2e/templates/cni-install.yml.j2

@@ -7,9 +7,9 @@ metadata:
 data:
   install_cni.sh: |
     cd /tmp
-    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
+    wget https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz
     cd /host/opt/cni/bin
-    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.4.0.tgz
     sleep infinite
 ---
 apiVersion: apps/v1

e2e/templates/dra-integration.yml.j2

@@ -0,0 +1,51 @@
+---
+apiVersion: resource.k8s.io/v1beta1
+kind: ResourceClaimTemplate
+metadata:
+  name: single-gpu
+spec:
+  spec:
+    devices:
+      requests:
+      - name: gpu
+        deviceClassName: gpu.example.com
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: dra-net
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: single-gpu
+spec:
+  config: '{
+        "cniVersion": "{{ CNI_VERSION }}",
+        "plugins": [{
+            "name": "mynet",
+            "type": "dummy",
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.1.2.0/24"
+            }
+        }]
+    }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dra-integration
+  labels:
+    app: dra-integration
+  annotations:
+    k8s.v1.cni.cncf.io/networks: default/dra-net
+spec:
+  containers:
+  - name: ctr0
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args: ["export; sleep 9999"]
+    resources:
+      claims:
+      - name: gpu
+  resourceClaims:
+  - name: gpu
+    resourceClaimTemplateName: single-gpu

e2e/templates/multus-daemonset-thick.yml.j2

@@ -43,7 +43,9 @@ rules:
       - pods/status
     verbs:
       - get
+      - list
       - update
+      - watch
   - apiGroups:
       - ""
       - events.k8s.io
@@ -131,7 +133,6 @@ spec:
       containers:
       - name: kube-multus
         image: localhost:5000/multus:e2e
-        imagePullPolicy: Always
         command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
         resources:
           requests:
@@ -157,13 +158,23 @@ spec:
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
           readOnly: true
+        - name: kubelet-pod-resources
+          mountPath: /var/lib/kubelet/pod-resources
+          readOnly: true
+        env:
+        - name: MULTUS_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       initContainers:
       - name: install-multus-shim
         image: localhost:5000/multus:e2e
         command:
-          - "cp"
-          - "/usr/src/multus-cni/bin/multus-shim"
-          - "/host/opt/cni/bin/multus-shim"
+          - "/usr/src/multus-cni/bin/install_multus"
+          - "-d"
+          - "/host/opt/cni/bin"
+          - "-t"
+          - "thick"
         resources:
           requests:
             cpu: "10m"
@@ -181,6 +192,9 @@ spec:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
+        - name: kubelet-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
         - name: multus-daemon-config
           configMap:
             name: multus-daemon-config

e2e/templates/subdirectory-chain-passthru-configupdate.yml.j2

@@ -0,0 +1,26 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "debug",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/host/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus",
+        "cniVersion": "{{ CNI_VERSION }}",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "forceCNIVersion": true,
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "auxiliaryCNIChainName": "vendor-cni-chain"
+    }

e2e/templates/subdirectory-chaining-passthru.yml.j2

@@ -0,0 +1,94 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cni-setup-script
+  namespace: default
+data:
+  setup.sh: |
+    #!/bin/bash
+    set -euxo pipefail
+
+    DEFAULT_NETWORK_CNI_NAME="vendor-cni-chain"
+
+    cleanup() {
+      echo "Cleaning up..."
+      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+      if [ $? -ne 0 ]; then
+        echo "Failed to remove sysctltwiddle.conf" >&2
+        exit 1
+      fi
+      echo "Cleanup completed successfully"
+    }
+    trap cleanup EXIT
+
+    # Create the chained CNI directory if it doesn't exist
+    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
+    if [ $? -ne 0 ]; then
+      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
+      exit 1
+    fi
+
+    # Write the chained tuning CNI config
+    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+    {
+      "cniVersion": "{{ CNI_VERSION }}",
+      "name": "sysctltwiddle",
+      "type": "tuning",
+      "sysctl": {
+        "net.ipv4.conf.eth0.arp_filter": "1"
+      }
+    }
+    EOF
+
+    if [ $? -ne 0 ]; then
+      echo "Failed to create chained CNI config" >&2
+      exit 1
+    fi
+
+    echo "CNI chained setup completed successfully."
+    sleep infinity
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cni-setup-daemonset
+  namespace: default
+  labels:
+    app: cni-setup
+spec:
+  selector:
+    matchLabels:
+      app: cni-setup
+  template:
+    metadata:
+      labels:
+        app: cni-setup
+    spec:
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      containers:
+      - name: setup
+        image: quay.io/fedora/fedora:40
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-config
+          mountPath: /host/etc/cni/net.d
+        - name: script-volume
+          mountPath: /scripts
+        command: ["/bin/bash", "/scripts/setup.sh"]
+      volumes:
+      - name: cni-config
+        hostPath:
+          path: /etc/cni/net.d
+          type: Directory
+      - name: script-volume
+        configMap:
+          name: cni-setup-script
+          items:
+          - key: setup.sh
+            path: setup.sh

e2e/templates/subdirectory-chaining-pod.yml.j2

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sysctl-modified
+spec:
+  containers:
+  - name: sysctl
+    image: quay.io/dosmith/fedora-procps
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    securityContext:
+      privileged: true

e2e/templates/subdirectory-chaining.yml.j2

@@ -0,0 +1,95 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cni-setup-script
+  namespace: default
+data:
+  setup.sh: |
+    #!/bin/bash
+    set -euxo pipefail
+
+    DEFAULT_NETWORK_CNI_NAME="kindnet"
+
+    cleanup() {
+      echo "Cleaning up..."
+      rm -f /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+      if [ $? -ne 0 ]; then
+        echo "Failed to remove sysctltwiddle.conf" >&2
+        exit 1
+      fi
+      echo "Cleanup completed successfully"
+    }
+    trap cleanup EXIT
+
+    # Create the chained CNI directory if it doesn't exist
+    mkdir -p /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}
+    if [ $? -ne 0 ]; then
+      echo "Failed to create directory /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}" >&2
+      exit 1
+    fi
+
+    # Write the chained tuning CNI config
+    cat <<EOF > /host/etc/cni/net.d/${DEFAULT_NETWORK_CNI_NAME}/sysctltwiddle.conf
+    {
+      "cniVersion": "{{ CNI_VERSION }}",
+      "name": "sysctltwiddle",
+      "type": "tuning",
+      "sysctl": {
+        "net.ipv4.conf.IFNAME.arp_filter": "1"
+      }
+    }
+    EOF
+
+    if [ $? -ne 0 ]; then
+      echo "Failed to create chained CNI config" >&2
+      exit 1
+    fi
+
+    echo "CNI chained setup completed successfully."
+    sleep infinity
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cni-setup-daemonset
+  namespace: default
+  labels:
+    app: cni-setup
+spec:
+  selector:
+    matchLabels:
+      app: cni-setup
+  template:
+    metadata:
+      labels:
+        app: cni-setup
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      containers:
+      - name: setup
+        image: quay.io/fedora/fedora:40
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-config
+          mountPath: /host/etc/cni/net.d
+        - name: script-volume
+          mountPath: /scripts
+        command: ["/bin/bash", "/scripts/setup.sh"]
+      volumes:
+      - name: cni-config
+        hostPath:
+          path: /etc/cni/net.d
+          type: Directory
+      - name: script-volume
+        configMap:
+          name: cni-setup-script
+          items:
+          - key: setup.sh
+            path: setup.sh

e2e/test-dra-integration.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# This test is using an example implementation of a DRA driver. This driver is mocking GPU resources. At our test we
+# don't care about what these resources are. We want to ensure that such resource is correctly passed in the Pod using
+# Multus configurations. A couple of notes:
+# - We explitictly pin the revision of the dra-example-driver to the branch `classic-dra` to indicate that the
+#   integration continues to work even when the dra-example-driver is updated. We know that classic-dra is supported
+#   in Kubernetes versions 1.26 to 1.30. Multus supports DRA in the aforementioned Kubernetes versions.
+# - The chart and latest is image is not published somewhere, therefore we have to build locally. This leads to slower
+#   e2e suite runs.
+echo "installing dra-example-driver"
+repo_path="repos/dra-example-driver"
+
+rm -rf $repo_path || true
+git clone --branch main https://github.com/kubernetes-sigs/dra-example-driver.git ${repo_path}
+MULTUS_DIR=$(pwd)
+cd ${repo_path}
+./demo/build-driver.sh
+KIND_CLUSTER_NAME=kind ./demo/scripts/load-driver-image-into-kind.sh
+cd "$MULTUS_DIR"
+chart_path=${repo_path}/deployments/helm/dra-example-driver/
+overriden_values_path=${chart_path}/overriden_values.yaml
+
+# With the thick plugin, in kind, the primary network on the control plane is not always working as expected. The pods
+# sometimes are not able to communicate with the control plane and the error looks like this:
+# failed to list *v1alpha2.PodSchedulingContext: Get "https://10.96.0.1:443/apis/resource.k8s.io/v1alpha2/podschedulingcontexts?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
+# We override the values here to schedule the controller on the worker nodes where the network is working as expected.
+cat <<EOF >> ${overriden_values_path}
+controller:
+  nodeSelector: null
+  tolerations: null
+EOF
+
+helm install \
+    -n dra-example-driver \
+    --create-namespace \
+    -f ${overriden_values_path} \
+    dra-example-driver \
+    ${chart_path}
+
+echo "installing testing pods"
+kubectl create -f yamls/dra-integration.yml
+kubectl wait --for=condition=ready -l app=dra-integration --timeout=300s pod
+
+echo "check dra-integration pod for DRA injected environment variable"
+
+# We can validate that the resource is correctly injected by checking an environment variable this dra driver is injecting
+# in the Pod.
+# https://github.com/kubernetes-sigs/dra-example-driver/blob/be2b8b1db47b8c757440e955ce5ced88c23bfe86/cmd/dra-example-kubeletplugin/cdi.go#L71C20-L71C44
+env_variable=$(kubectl exec dra-integration -- bash -c "echo \$DRA_RESOURCE_DRIVER_NAME | grep gpu.example.com")
+if [ $? -eq 0 ];then
+	echo "dra-integration pod has DRA injected environment variable"
+else
+	echo "dra-integration pod doesn't have DRA injected environment variable"
+	exit 1
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/dra-integration.yml
+helm uninstall -n dra-example-driver dra-example-driver

e2e/test-subdirectory-chaining-passthru.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${PATH}:./bin
+
+TEST_POD_NAME="sysctl-modified"
+EXPECTED_BINARIES="${EXPECTED_BINARIES:-/opt/cni/bin/ptp /opt/cni/bin/portmap /opt/cni/bin/tuning}"
+EXPECTED_CNI_DIR="/etc/cni/net.d"
+
+# Reconfigure multus
+echo "Applying subdirectory chain passthru config..."
+kubectl apply -f yamls/subdirectory-chain-passthru-configupdate.yml
+
+# Restart the multus daemonset to pick up the new config
+echo "Restarting Multus DaemonSet..."
+kubectl rollout restart daemonset kube-multus-ds-amd64 -n kube-system
+kubectl rollout status daemonset/kube-multus-ds-amd64 -n kube-system
+
+# Debug: show CNI configs and binaries inside each Kind node
+echo "Checking CNI configs and binaries on nodes..."
+
+for node in $(kubectl get nodes --no-headers | awk '{print $1}'); do
+    container_name=$(docker ps --format '{{.Names}}' | grep "^${node}$")
+
+    echo "------"
+    echo "Node: ${node} (container: ${container_name})"
+    echo "Listing /opt/cni/bin contents..."
+    docker exec "${container_name}" ls -l /opt/cni/bin || echo "WARNING: /opt/cni/bin missing!"
+
+    echo "Checking expected binaries..."
+    for bin in $EXPECTED_BINARIES; do
+        echo "Checking for ${bin}..."
+        if docker exec "${container_name}" test -f "${bin}"; then
+            echo "SUCCESS: ${bin} found."
+        else
+            echo "FAIL: ${bin} NOT found!"
+        fi
+    done
+
+    echo "Listing /etc/cni/net.d configs..."
+    docker exec "${container_name}" ls -l ${EXPECTED_CNI_DIR} || echo "WARNING: ${EXPECTED_CNI_DIR} missing!"
+done
+echo "------"
+
+# Deploy the daemonset that will lay down the chained CNI config
+echo "Applying CNI setup DaemonSet..."
+kubectl apply -f yamls/subdirectory-chaining-passthru.yml
+
+# Wait for the daemonset pods to be ready (make sure they set up CNI config)
+echo "Waiting for CNI setup DaemonSet to be Ready..."
+kubectl rollout status daemonset/cni-setup-daemonset --timeout=300s
+
+# Deploy a test pod that will get chained CNI applied
+echo "Applying test pod..."
+kubectl apply -f yamls/subdirectory-chaining-pod.yml
+
+# Wait for the pod to be Ready
+echo "Waiting for test pod to be Ready..."
+kubectl wait --for=condition=ready pod/${TEST_POD_NAME} --timeout=300s
+
+# Check that the sysctl got set
+echo "Verifying sysctl arp_filter is set to 1 on eth0..."
+
+SYSCTL_VALUE=$(kubectl exec ${TEST_POD_NAME} -- sysctl -n net.ipv4.conf.eth0.arp_filter)
+
+if [ "$SYSCTL_VALUE" != "1" ]; then
+  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
+  exit 1
+else
+  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
+fi
+
+# Cleanup
+echo "Cleaning up test resources..."
+kubectl delete -f yamls/subdirectory-chaining-pod.yml
+kubectl delete -f yamls/subdirectory-chaining-passthru.yml
+
+echo "Test completed successfully."
+exit 0

e2e/test-subdirectory-chaining.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+TEST_POD_NAME="sysctl-modified"
+
+# Deploy the daemonset that will lay down the chained CNI config
+kubectl apply -f yamls/subdirectory-chaining.yml
+
+# Wait for the daemonset pods to be ready (we need the config to be laid down)
+kubectl rollout status daemonset/cni-setup-daemonset
+
+# Deploy a test pod that will get chained CNI applied
+kubectl apply -f yamls/subdirectory-chaining-pod.yml
+
+# Wait for the pod to be Ready
+kubectl wait --for=condition=ready pod/sysctl-modified --timeout=300s
+
+# Check that the sysctl got set properly inside the pod's eth0 interface
+echo "Verifying sysctl arp_filter is set to 1 on eth0"
+
+SYSCTL_VALUE=$(kubectl exec sysctl-modified -- sysctl -n net.ipv4.conf.eth0.arp_filter)
+
+if [ "$SYSCTL_VALUE" != "1" ]; then
+  echo "FAIL: net.ipv4.conf.eth0.arp_filter is not set to 1, got ${SYSCTL_VALUE}" >&2
+  exit 1
+else
+  echo "SUCCESS: net.ipv4.conf.eth0.arp_filter is set correctly."
+fi
+
+# 6. Clean up
+echo "Cleaning up test resources"
+kubectl delete -f yamls/subdirectory-chaining-pod.yml
+kubectl delete -f yamls/subdirectory-chaining.yml
+
+exit 0

examples/macvlan-pod.yml

@@ -54,3 +54,4 @@ spec:
     image: dougbtv/centos-network
     ports:
     - containerPort: 80
+  automountServiceAccountToken: false

examples/sriov-pod.yml

@@ -45,3 +45,4 @@ spec:
       limits:
         intel.com/sriov: '1'
   restartPolicy: "Never"
+  automountServiceAccountToken: false

go.mod

@@ -1,96 +1,77 @@
 module gopkg.in/k8snetworkplumbingwg/multus-cni.v4
 
-go 1.18
+go 1.24.11
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/containernetworking/cni v1.1.2
-	github.com/containernetworking/plugins v1.1.0
-	github.com/fsnotify/fsnotify v1.6.0
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
-	github.com/onsi/ginkgo/v2 v2.5.1
-	github.com/onsi/gomega v1.24.0
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
-	golang.org/x/net v0.7.0
-	golang.org/x/sys v0.5.0
-	google.golang.org/grpc v1.40.0
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	k8s.io/api v0.22.8
-	k8s.io/apimachinery v0.22.8
-	k8s.io/client-go v0.22.8
+	github.com/containernetworking/cni v1.3.0
+	github.com/containernetworking/plugins v1.9.0
+	github.com/fsnotify/fsnotify v1.9.0
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6
+	github.com/onsi/ginkgo/v2 v2.25.1
+	github.com/onsi/gomega v1.38.1
+	github.com/prometheus/client_golang v1.22.0
+	github.com/spf13/pflag v1.0.6
+	github.com/vishvananda/netlink v1.3.1
+	golang.org/x/net v0.43.0
+	golang.org/x/sys v0.35.0
+	google.golang.org/grpc v1.73.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	k8s.io/api v0.34.1
+	k8s.io/apimachinery v0.34.1
+	k8s.io/client-go v0.34.1
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.60.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 // indirect
-	k8s.io/kubelet v0.22.8
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	k8s.io/klog/v2 v2.130.1
+	k8s.io/kubelet v0.34.1
 )
 
-require github.com/prometheus/client_golang v1.12.2
-
 require (
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/oauth2 v0.28.0 // indirect
+	golang.org/x/term v0.34.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/protobuf v1.36.7 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-)
-
-replace (
-	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
-	k8s.io/api => k8s.io/api v0.22.8
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.8
-	k8s.io/apimachinery => k8s.io/apimachinery v0.22.8
-	k8s.io/apiserver => k8s.io/apiserver v0.22.8
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.22.8
-	k8s.io/client-go => k8s.io/client-go v0.22.8
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.22.8
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.8
-	k8s.io/code-generator => k8s.io/code-generator v0.22.8
-	k8s.io/component-base => k8s.io/component-base v0.22.8
-	k8s.io/component-helpers => k8s.io/component-helpers v0.22.8
-	k8s.io/controller-manager => k8s.io/controller-manager v0.22.8
-	k8s.io/cri-api => k8s.io/cri-api v0.22.8
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.22.8
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.22.8
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.22.8
-	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.22.8
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.22.8
-	k8s.io/kubectl => k8s.io/kubectl v0.22.8
-	k8s.io/kubelet => k8s.io/kubelet v0.22.8
-	k8s.io/kubernetes => k8s.io/kubernetes v1.22.8
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.22.8
-	k8s.io/metrics => k8s.io/metrics v0.22.8
-	k8s.io/mount-utils => k8s.io/mount-utils v0.22.8
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.8
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.22.8
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -1,810 +1,230 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
-github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
-github.com/containernetworking/plugins v1.1.0 h1:kTIldaDo9SlbQsjhUKvDx0v9q7zyIFJH/Rm9F4xRBro=
-github.com/containernetworking/plugins v1.1.0/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo=
+github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4=
+github.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6LC3U+QrEANuQ=
+github.com/containernetworking/plugins v1.9.0/go.mod h1:JG3BxoJifxxHBhG3hFyxyhid7JgRVBu/wtooGEvWf1c=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
+github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=
+github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 h1:lhSaboKtal0XF2yqSw2BqNB1vUL4+a4BFe39I9G/yiM=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw=
-github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg=
-github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.25.1 h1:Fwp6crTREKM+oA6Cz4MsO8RhKQzs2/gOIVOUscMAfZY=
+github.com/onsi/ginkgo/v2 v2.25.1/go.mod h1:ppTWQ1dh9KM/F1XgpeRqelR+zHVwV81DGRSDnFxK7Sk=
+github.com/onsi/gomega v1.38.1 h1:FaLA8GlcpXDwsb7m0h2A9ew2aTk3vnZMlzFgg5tz/pk=
+github.com/onsi/gomega v1.38.1/go.mod h1:LfcV8wZLvwcYRwPiJysphKAEsmcFnLMK/9c+PjvlX8g=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA=
-github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
-github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
+github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
+github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
+github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
-go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
-go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
-go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
-go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
-go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
-go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
-go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
-go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
+go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc=
+golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
+golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
+golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
+google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
+google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
+google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.22.8 h1:7Ld6tHuvaYzcQE2axLmomWlhP0fK3vpLfo6fBaNrCIs=
-k8s.io/api v0.22.8/go.mod h1:uLlWJNRJ+AYwgAdsNwf0TsD3eByNYW9RlXFmkMdL3yk=
-k8s.io/apimachinery v0.22.8 h1:kazMo4/t5ZPI7MwImnCJODZrt1VuwbYBixhTzaNIxsw=
-k8s.io/apimachinery v0.22.8/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU=
-k8s.io/client-go v0.22.8 h1:dWgwPqpWH/DPLWSczA6b61VxFIILe989MXipoE9332s=
-k8s.io/client-go v0.22.8/go.mod h1:dOHOy82WOBz0siYHpVyY7FqTIq+iXFXW3+THFk6qErU=
-k8s.io/component-base v0.22.8/go.mod h1:rt0sHx3GT3i8e72rPKQSrTIV3THWyEl2IDYcnEbxraI=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM=
+k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk=
+k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4=
+k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw=
+k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY=
+k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
-k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kubelet v0.22.8 h1:Ce1yoWCd03Gp5Dqe4BUy1FlQntyIUfCUcL8E6mKIgi4=
-k8s.io/kubelet v0.22.8/go.mod h1:IHfh4AnROFQ6fMnCeoTQIrnF0H148PlfllXeupW698c=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts=
+k8s.io/kubelet v0.34.1 h1:doAaTA9/Yfzbdq/u/LveZeONp96CwX9giW6b+oHn4m4=
+k8s.io/kubelet v0.34.1/go.mod h1:PtV3Ese8iOM19gSooFoQT9iyRisbmJdAPuDImuccbbA=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

hack/build-go.sh

@@ -57,22 +57,28 @@ LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version=${VER
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.gitTreeState=${GIT_TREE_STATE} \
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.releaseStatus=${RELEASE_STATUS} \
 	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date=${DATE}"
-export CGO_ENABLED=0
+export CGO_ENABLED=${CGO_ENABLED:-0}
 
 # build with go modules
 export GO111MODULE=on
-BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+
 if [ -n "$MODMODE" ]; then
-	BUILD_ARGS+=(-mod "$MODMODE")
+	BUILD_ARGS=(-mod "$MODMODE")
 fi
 
 echo "Building multus"
-go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
+go build -o ${DEST_DIR}/multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
 echo "Building multus-daemon"
-go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/multus-daemon
+go build -o "${DEST_DIR}"/multus-daemon ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-daemon
 echo "Building multus-shim"
-go build -o "${DEST_DIR}"/multus-shim -ldflags "${LDFLAGS}" ./cmd/multus-shim
+go build -o "${DEST_DIR}"/multus-shim ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-shim
 echo "Building install_multus"
-go build -o "${DEST_DIR}"/install_multus -ldflags "${LDFLAGS}" ./cmd/install_multus
+go build -o "${DEST_DIR}"/install_multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/install_multus
 echo "Building thin_entrypoint"
-go build -o "${DEST_DIR}"/thin_entrypoint -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
+go build -o "${DEST_DIR}"/thin_entrypoint ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
+echo "Building kubeconfig_generator"
+go build -o "${DEST_DIR}"/kubeconfig_generator ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/kubeconfig_generator
+echo "Building cert-approver"
+go build -o "${DEST_DIR}"/cert-approver ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/cert-approver
+echo "Building passthru CNI"
+go build -o "${DEST_DIR}"/passthru ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/passthru-cni

hack/test-go.sh

@@ -19,5 +19,5 @@ if [ "$GO111MODULE" == "off" ]; then
 	bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
 else
 	# test with go modules
-	bash -c "umask 0; go test -v -covermode=count -coverprofile=coverage.out ./..."
+	bash -c "umask 0; go test -v -race -covermode=atomic -coverprofile=coverage.out ./..."
 fi

images/Dockerfile

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.24 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -8,7 +8,7 @@ ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM gcr.io/distroless/base-debian11:latest
+FROM gcr.io/distroless/base-debian12:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
@@ -16,4 +16,7 @@ WORKDIR /
 
 COPY --from=build /usr/src/multus-cni/bin/install_multus /
 COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+
 ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.debug

@@ -1,5 +1,5 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM --platform=$BUILDPLATFORM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.24 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
@@ -8,7 +8,7 @@ ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
-FROM gcr.io/distroless/base-debian11:debug
+FROM gcr.io/distroless/base-debian12:debug
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
@@ -16,4 +16,7 @@ WORKDIR /
 
 COPY --from=build /usr/src/multus-cni/bin/install_multus /
 COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+
 ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.thick

@@ -1,16 +1,18 @@
 # This Dockerfile is used to build the image available on DockerHub
-FROM golang:1.19 as build
+FROM --platform=$BUILDPLATFORM golang:1.24 as build
 
 # Add everything
 ADD . /usr/src/multus-cni
 
+ARG TARGETPLATFORM
 RUN  cd /usr/src/multus-cni && \
      ./hack/build-go.sh
 
 FROM debian:stable-slim
-LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+LABEL org.opencontainers.image.source=https://github.com/k8snetworkplumbingwg/multus-cni
 COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
 COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
 WORKDIR /
 
 ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

pkg/checkpoint/checkpoint_test.go

@@ -15,6 +15,8 @@
 
 package checkpoint
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"

pkg/cmdutils/cmdutils_suite_test.go

@@ -15,6 +15,8 @@
 // Package cmdutils is the package that contains utilities for multus command
 package cmdutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/cmdutils/utils_test.go

@@ -15,6 +15,8 @@
 // Package cmdutils is the package that contains utilities for multus command
 package cmdutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"

pkg/k8sclient/k8sclient.go

@@ -18,29 +18,29 @@ package k8sclient
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
+	"syscall"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
+	listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/klog"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
-	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1"
 	netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/kubeletclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
@@ -61,9 +61,13 @@ type NoK8sNetworkError struct {
 // ClientInfo contains information given from k8s client
 type ClientInfo struct {
 	Client           kubernetes.Interface
-	NetClient        netclient.K8sCniCncfIoV1Interface
+	NetClient        netclient.Interface
 	EventBroadcaster record.EventBroadcaster
 	EventRecorder    record.EventRecorder
+
+	// multus-thick uses these informer
+	PodInformer    cache.SharedIndexInformer
+	NetDefInformer cache.SharedIndexInformer
 }
 
 // AddPod adds pod into kubernetes
@@ -73,9 +77,27 @@ func (c *ClientInfo) AddPod(pod *v1.Pod) (*v1.Pod, error) {
 
 // GetPod gets pod from kubernetes
 func (c *ClientInfo) GetPod(namespace, name string) (*v1.Pod, error) {
+	if c.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
+		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
+	}
 	return c.Client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
+// GetPodContext gets pod from kubernetes with context
+func (c *ClientInfo) GetPodContext(ctx context.Context, namespace, name string) (*v1.Pod, error) {
+	if c.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", namespace, name)
+		return listers.NewPodLister(c.PodInformer.GetIndexer()).Pods(namespace).Get(name)
+	}
+	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+}
+
+// GetPodAPILiveQuery does a live API query for the pod, instead of using informers, for cases when a failure occurred, as to prevent a cache miss.
+func (c *ClientInfo) GetPodAPILiveQuery(ctx context.Context, namespace, name string) (*v1.Pod, error) {
+	return c.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+}
+
 // DeletePod deletes a pod from kubernetes
 func (c *ClientInfo) DeletePod(namespace, name string) error {
 	return c.Client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
@@ -83,7 +105,16 @@ func (c *ClientInfo) DeletePod(namespace, name string) error {
 
 // AddNetAttachDef adds net-attach-def into kubernetes
 func (c *ClientInfo) AddNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) (*nettypes.NetworkAttachmentDefinition, error) {
-	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
+	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
+}
+
+// GetNetAttachDef get net-attach-def from kubernetes
+func (c *ClientInfo) GetNetAttachDef(namespace, name string) (*nettypes.NetworkAttachmentDefinition, error) {
+	if c.NetDefInformer != nil {
+		logging.Debugf("GetNetAttachDef for [%s/%s] will use informer cache", namespace, name)
+		return netlister.NewNetworkAttachmentDefinitionLister(c.NetDefInformer.GetIndexer()).NetworkAttachmentDefinitions(namespace).Get(name)
+	}
+	return c.NetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 }
 
 // Eventf puts event into kubernetes events
@@ -113,7 +144,7 @@ func SetPodNetworkStatusAnnotation(client *ClientInfo, podName string, podNamesp
 	if err != nil {
 		return logging.Errorf("SetNetworkStatus: %v", err)
 	}
-	if client == nil || client.Client == nil {
+	if client == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
 			return logging.Errorf("SetNetworkStatus: must have either Kubernetes config or delegates")
@@ -168,13 +199,19 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	// Check and see if each item matches the specification for valid attachment name.
 	// "Valid attachment names must be comprised of units of the DNS-1123 label format"
 	// [a-z0-9]([-a-z0-9]*[a-z0-9])?
-	// And we allow at (@), and forward slash (/) (units separated by commas)
 	// It must start and end alphanumerically.
-	allItems := []string{netNsName, networkName, netIfName}
+	allItems := []string{netNsName, networkName}
+	expr := regexp.MustCompile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
 	for i := range allItems {
-		matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i])
+		matched := expr.MatchString(allItems[i])
 		if !matched && len([]rune(allItems[i])) > 0 {
-			return "", "", "", logging.Errorf(fmt.Sprintf("parsePodNetworkObjectName: Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
+			return "", "", "", logging.Errorf("parsePodNetworkObjectName: Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i])
+		}
+	}
+
+	if len(netIfName) > 0 {
+		if len(netIfName) > (syscall.IFNAMSIZ-1) || strings.ContainsAny(netIfName, " \t\n\v\f\r/") {
+			return "", "", "", logging.Errorf("parsePodNetworkObjectName: Failed to parse interface name: must be less than 15 chars and not contain '/' or spaces. interface name '%s'", netIfName)
 		}
 	}
 
@@ -254,19 +291,20 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
 
 	logging.Debugf("getKubernetesDelegate: %v, %v, %s, %v, %v", client, net, confdir, pod, resourceMap)
-	customResource, err := client.NetClient.NetworkAttachmentDefinitions(net.Namespace).Get(context.TODO(), net.Name, metav1.GetOptions{})
+
+	customResource, err := client.GetNetAttachDef(net.Namespace, net.Name)
 	if err != nil {
 		errMsg := fmt.Sprintf("cannot find a network-attachment-definition (%s) in namespace (%s): %v", net.Name, net.Namespace, err)
 		if client != nil {
 			client.Eventf(pod, v1.EventTypeWarning, "NoNetworkFound", errMsg)
 		}
-		return nil, resourceMap, logging.Errorf("getKubernetesDelegate: " + errMsg)
+		return nil, resourceMap, logging.Errorf("getKubernetesDelegate: %s", errMsg)
 	}
 
 	// Get resourceName annotation from NetworkAttachmentDefinition
 	deviceID := ""
 	resourceName, ok := customResource.GetAnnotations()[resourceNameAnnot]
-	if ok && pod.Name != "" && pod.Namespace != "" {
+	if ok && pod != nil && pod.Name != "" && pod.Namespace != "" {
 		// ResourceName annotation is found; try to get device info from resourceMap
 		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
 
@@ -292,11 +330,6 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen
 		}
 	}
 
-	// acquire lock to access file
-	if types.ChrootMutex != nil {
-		types.ChrootMutex.Lock()
-		defer types.ChrootMutex.Unlock()
-	}
 	configBytes, err := netutils.GetCNIConfig(customResource, confdir)
 	if err != nil {
 		return nil, resourceMap, err
@@ -393,82 +426,6 @@ func TryLoadPodDelegates(pod *v1.Pod, conf *types.NetConf, clientInfo *ClientInf
 	return 0, clientInfo, err
 }
 
-// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
-// the k8s API.
-func InClusterK8sClient() (*ClientInfo, error) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	logging.Debugf("InClusterK8sClient: in cluster config: %+v", config)
-	return NewClientInfo(config)
-}
-
-// GetK8sClient gets client info from kubeconfig
-func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
-	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
-	// If we get a valid kubeClient (eg from testcases) just return that
-	// one.
-	if kubeClient != nil {
-		return kubeClient, nil
-	}
-
-	var err error
-	var config *rest.Config
-
-	// Otherwise try to create a kubeClient from a given kubeConfig
-	if kubeconfig != "" {
-		// uses the current context in kubeconfig
-		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
-		}
-	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
-		// Try in-cluster config where multus might be running in a kubernetes pod
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
-		}
-	} else {
-		// No kubernetes config; assume we shouldn't talk to Kube at all
-		return nil, nil
-	}
-
-	// Specify that we use gRPC
-	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
-	config.ContentType = "application/vnd.kubernetes.protobuf"
-	// Set the config timeout to one minute.
-	config.Timeout = time.Minute
-
-	return NewClientInfo(config)
-}
-
-// NewClientInfo returns a `ClientInfo` from a configuration created from an
-// existing kubeconfig file.
-func NewClientInfo(config *rest.Config) (*ClientInfo, error) {
-	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	netclient, err := netclient.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(klog.Infof)
-	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
-	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
-	return &ClientInfo{
-		Client:           client,
-		NetClient:        netclient,
-		EventBroadcaster: broadcaster,
-		EventRecorder:    recorder,
-	}, nil
-}
-
 // GetPodNetwork gets net-attach-def annotation from pod
 func GetPodNetwork(pod *v1.Pod) ([]*types.NetworkSelectionElement, error) {
 	logging.Debugf("GetPodNetwork: %v", pod)
@@ -548,12 +505,6 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 
 		// option2) search CNI json config file, which has <netname> as CNI name, from confDir
 
-		// acquire lock to access file
-		if types.ChrootMutex != nil {
-			types.ChrootMutex.Lock()
-			defer types.ChrootMutex.Unlock()
-		}
-
 		configBytes, err = netutils.GetCNIConfigFromFile(netname, confdir)
 		if err == nil {
 			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
@@ -563,12 +514,6 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 			return delegate, resourceMap, nil
 		}
 	} else {
-		// acquire lock to access file
-		if types.ChrootMutex != nil {
-			types.ChrootMutex.Lock()
-			defer types.ChrootMutex.Unlock()
-		}
-
 		fInfo, err := os.Stat(netname)
 		if err != nil {
 			return nil, resourceMap, err
@@ -595,31 +540,119 @@ func getNetDelegate(client *ClientInfo, pod *v1.Pod, netname, confdir, namespace
 		} else {
 			// option4) if file path (absolute), then load it directly
 			if strings.HasSuffix(netname, ".conflist") {
-				confList, err := libcni.ConfListFromFile(netname)
+				confList, err := LoadChainedPluginsFromFile(netname)
 				if err != nil {
 					return nil, resourceMap, logging.Errorf("error loading CNI conflist file %s: %v", netname, err)
 				}
-				configBytes = confList.Bytes
-			} else {
-				conf, err := libcni.ConfFromFile(netname)
+
+				delegate, err := types.LoadDelegateNetConfFromConfList(confList, nil, "", "")
 				if err != nil {
-					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
+					return nil, resourceMap, err
 				}
-				if conf.Network.Type == "" {
-					return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", netname)
-				}
-				configBytes = conf.Bytes
+				return delegate, resourceMap, nil
+
 			}
-			delegate, err := types.LoadDelegateNetConf(configBytes, nil, "", "")
+
+			// Or it's not a conflist...
+			// after libcni v1.2.3 there's no support support this old-school method with non-conflists.
+			// this method doesn't check if there's a 0 length plugins field, that is.
+			conf, err := libcni.ConfFromFile(netname)
+			if err != nil {
+				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: %v", netname, err)
+			}
+			if conf.Network.Type == "" {
+				return nil, resourceMap, logging.Errorf("error loading CNI config file %s: no 'type'; perhaps this is supposed to be a .conflist?", netname)
+			}
+
+			delegate, err := types.LoadDelegateNetConf(conf.Bytes, nil, "", "")
 			if err != nil {
 				return nil, resourceMap, err
 			}
 			return delegate, resourceMap, nil
 		}
+
 	}
 	return nil, resourceMap, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
 }
 
+func loadSubdirectoryChain(bytes []byte, cniconfdir string) (*libcni.NetworkConfigList, error) {
+	// Load the network configuration from the byte array
+	conf, err := libcni.NetworkConfFromBytes(bytes)
+	if err != nil {
+		return nil, fmt.Errorf("error loading network config from bytes: %v", err)
+	}
+
+	// Check if plugins need to be loaded from files
+	if !conf.LoadOnlyInlinedPlugins && cniconfdir != "" {
+		// Let's validate that conf.Name
+		// From the CNI spec:
+		// > Must start with an alphanumeric character, optionally followed by any combination of one or more alphanumeric characters,
+		// > underscore, dot (.) or hyphen (-). Must not contain characters disallowed in file paths.
+		if !regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`).MatchString(conf.Name) {
+			return nil, fmt.Errorf("invalid network config name: %s", conf.Name)
+		}
+
+		plugins, err := libcni.NetworkPluginConfsFromFiles(cniconfdir, conf.Name)
+		if err != nil {
+			return nil, fmt.Errorf("error loading plugin configs: %v", err)
+		}
+		conf.Plugins = append(conf.Plugins, plugins...)
+	}
+
+	if len(conf.Plugins) == 0 {
+		return nil, fmt.Errorf("no plugin configs found")
+	}
+
+	return conf, nil
+}
+
+// LoadChainedDelegatesFromBytes loads a CNI configuration byte array and returns a DelegateNetConf with the chain added.
+func LoadChainedDelegatesFromBytes(bytes []byte, cniconfdir string) *types.DelegateNetConf {
+	conf, err := loadSubdirectoryChain(bytes, cniconfdir)
+	if err != nil {
+		logging.Errorf("LoadChainedDelegatesFromBytes: %v", err)
+		return nil
+	}
+
+	// Create and return a DelegateNetConf from the configuration list
+	delegate, err := types.LoadDelegateNetConfFromConfList(conf, nil, "", "")
+	if err != nil {
+		logging.Errorf("LoadChainedDelegatesFromBytes: error loading delegate network config: %v", err)
+		return nil
+	}
+
+	return delegate
+}
+
+// LoadChainedPluginsFromFile loads a CNI configuration file and returns the NetworkConfigList
+func LoadChainedPluginsFromFile(filename string) (*libcni.NetworkConfigList, error) {
+	cleanPath := filepath.Clean(filename)
+
+	// stat the file to make sure it's a normal file.
+	info, err := os.Stat(cleanPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.Mode().IsRegular() {
+		return nil, errors.New("CNI configuration path is not a regular file")
+	}
+
+	bytes, err := os.ReadFile(cleanPath)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", filename, err)
+	}
+	logging.Debugf("LoadChainedPluginsFromFile: %s", filename)
+
+	conf, err := loadSubdirectoryChain(bytes, filepath.Dir(filename))
+	if err != nil {
+		return nil, err
+	}
+	logging.Debugf("Loaded SubdirectoryChain: %+v", conf)
+
+	return conf, nil
+}
+
 // GetDefaultNetworks parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
 func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo, resourceMap map[string]*types.ResourceInfo) (map[string]*types.ResourceInfo, error) {
 	logging.Debugf("GetDefaultNetworks: %v, %v, %v, %v", pod, conf, kubeClient, resourceMap)
@@ -646,7 +679,7 @@ func GetDefaultNetworks(pod *v1.Pod, conf *types.NetConf, kubeClient *ClientInfo
 	delegates = append(delegates, delegate)
 
 	// Pod in kube-system namespace does not have default network for now.
-	if !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
+	if pod != nil && !types.CheckSystemNamespaces(pod.ObjectMeta.Namespace, conf.SystemNamespaces) {
 		for _, netname := range conf.DefaultNetworks {
 			delegate, resourceMap, err := getNetDelegate(kubeClient, pod, netname, conf.ConfDir, conf.MultusNamespace, resourceMap)
 			if err != nil {

pkg/k8sclient/k8sclient_test.go

@@ -15,6 +15,8 @@
 
 package k8sclient
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
@@ -46,7 +48,7 @@ func TestK8sClient(t *testing.T) {
 func NewFakeClientInfo() *ClientInfo {
 	return &ClientInfo{
 		Client:    fake.NewSimpleClientset(),
-		NetClient: netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient: netfake.NewSimpleClientset(),
 	}
 }
 
@@ -1000,36 +1002,30 @@ users:
 	})
 
 	Context("parsePodNetworkObjectName", func() {
-		It("fails to get podnetwork given bad annotation values", func() {
-			fakePod := testutils.NewFakePod(fakePodName, "net1", "")
-
-			clientInfo := NewFakeClientInfo()
-			_, err := clientInfo.AddPod(fakePod)
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = clientInfo.AddNetAttachDef(
-				testutils.NewFakeNetAttachDef(fakePod.ObjectMeta.Namespace, "net1", "{\"type\": \"mynet\"}"))
-			Expect(err).NotTo(HaveOccurred())
-
-			k8sArgs, err := GetK8sArgs(args)
-			Expect(err).NotTo(HaveOccurred())
-			pod, err := clientInfo.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
-			// invalid case 1 - can't have more than 2 items separated by "/"
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP/root@thirdIP"
+		DescribeTable("fails to get podnetwork given bad annotation values", func(networkAnnot string) {
+			pod := testutils.NewFakePod(fakePodName, "net1", "")
+			pod.Annotations[networkAttachmentAnnot] = networkAnnot
 			_, err = GetPodNetwork(pod)
 			Expect(err).To(HaveOccurred())
+		},
+			Entry("can't have more than 2 items separated by \"/\"", "root@someIP/root@someOtherIP/root@thirdIP"),
+			Entry("can't have more than 2 items separated by \"@\"", "root@someIP/root@someOtherIP@garbagevalue"),
+			Entry("not matching comma-delimited format", "root@someIP/root@someOtherIP"),
+			Entry("invalid network interface name space in netdev name", "default/net1@myIfc Name"),
+			Entry("invalid network interface name too long", "default/net1@very_long_interface_name"),
+		)
 
-			// invalid case 2 - can't have more than 2 items separated by "@"
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP@garbagevalue"
+		DescribeTable("gets pod network successfully from annotation values", func(networkAnnot string) {
+			pod := testutils.NewFakePod(fakePodName, "net1", "")
+			pod.Annotations[networkAttachmentAnnot] = networkAnnot
 			_, err = GetPodNetwork(pod)
-			Expect(err).To(HaveOccurred())
-
-			// invalid case 3 - not matching comma-delimited format
-			pod.Annotations[networkAttachmentAnnot] = "root@someIP/root@someOtherIP"
-			_, err = GetPodNetwork(pod)
-			Expect(err).To(HaveOccurred())
-		})
+			Expect(err).ToNot(HaveOccurred())
+		},
+			Entry("network without namespace", "net1"),
+			Entry("network with namespace", "default/net1"),
+			Entry("network with interface name", "net1@my_interface"),
+			Entry("network with interface name and namespace", "default/net1@my_interface"),
+		)
 	})
 
 	Context("setPodNetworkAnnotation", func() {
@@ -1203,11 +1199,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1258,11 +1257,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1316,11 +1318,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1398,11 +1403,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1454,11 +1462,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 
@@ -1509,11 +1520,14 @@ users:
 			delegate, err := types.LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 			Expect(err).NotTo(HaveOccurred())
 
-			delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
-			GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+			delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+			GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 			Expect(err).NotTo(HaveOccurred())
 
-			netstatus := []nettypes.NetworkStatus{*delegateNetStatus}
+			netstatus := make([]nettypes.NetworkStatus, 0)
+			for _, status := range delegateNetStatuses {
+				netstatus = append(netstatus, *status)
+			}
 
 			fakePod := testutils.NewFakePod(fakePodName, "kube-system/net1", "")
 

pkg/k8sclient/kubeconfig.go

@@ -0,0 +1,247 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8sclient
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/transport"
+	"k8s.io/client-go/util/certificate"
+	"k8s.io/klog"
+
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+)
+
+const (
+	certNamePrefix       = "multus-client"
+	certCommonNamePrefix = "system:multus"
+	certOrganization     = "system:multus"
+)
+
+var (
+	certUsages = []certificatesv1.KeyUsage{certificatesv1.UsageDigitalSignature, certificatesv1.UsageClientAuth}
+)
+
+// getPerNodeKubeconfig creates new kubeConfig, based on bootstrap, with new certDir
+func getPerNodeKubeconfig(bootstrap *rest.Config, certDir string) *rest.Config {
+	return &rest.Config{
+		Host:    bootstrap.Host,
+		APIPath: bootstrap.APIPath,
+		ContentConfig: rest.ContentConfig{
+			AcceptContentTypes: "application/vnd.kubernetes.protobuf,application/json",
+			ContentType:        "application/vnd.kubernetes.protobuf",
+		},
+		TLSClientConfig: rest.TLSClientConfig{
+			KeyFile:  path.Join(certDir, certNamePrefix+"-current.pem"),
+			CertFile: path.Join(certDir, certNamePrefix+"-current.pem"),
+			CAData:   bootstrap.TLSClientConfig.CAData,
+		},
+		// Allow multus (especially in server mode) to make more concurrent requests
+		// to reduce client-side throttling
+		QPS:   50,
+		Burst: 50,
+		// Set the config timeout to one minute.
+		Timeout: time.Minute,
+	}
+}
+
+// PerNodeK8sClient creates/reload new multus kubeconfig per-node.
+func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration time.Duration, certDir string) (*ClientInfo, error) {
+	bootstrapKubeconfig, err := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+	if err != nil {
+		return nil, logging.Errorf("failed to load bootstrap kubeconfig %s: %v", bootstrapKubeconfigFile, err)
+	}
+	config := getPerNodeKubeconfig(bootstrapKubeconfig, certDir)
+
+	// If we have a valid certificate, user that to fetch CSRs.
+	// Otherwise, use the bootstrap credentials from bootstrapKubeconfig
+	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
+	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
+		cfg := bootstrapKubeconfig
+
+		// validate the kubeconfig
+		tempClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
+		} else {
+			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+			// tls unknown authority error is unrecoverable error with retry
+			if err != nil {
+				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
+					// reload and use bootstrapKubeconfig again
+					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					cfg = newBootstrapKubeconfig
+				} else {
+					logging.Errorf("failed to list pods with new certs: %v", err)
+				}
+			}
+
+			if current != nil {
+				cfg = config
+			}
+		}
+		return kubernetes.NewForConfig(cfg)
+	}
+
+	certificateStore, err := certificate.NewFileStore(certNamePrefix, certDir, certDir, "", "")
+	if err != nil {
+		return nil, logging.Errorf("failed to initialize the certificate store: %v", err)
+	}
+
+	certManager, err := certificate.NewManager(&certificate.Config{
+		ClientsetFn: newClientsetFn,
+		Template: &x509.CertificateRequest{
+			Subject: pkix.Name{
+				CommonName:   fmt.Sprintf("%s:%s", certCommonNamePrefix, nodeName),
+				Organization: []string{certOrganization},
+			},
+		},
+		RequestedCertificateLifetime: &certDuration,
+		SignerName:                   certificatesv1.KubeAPIServerClientSignerName,
+		Usages:                       certUsages,
+		CertificateStore:             certificateStore,
+	})
+	if err != nil {
+		return nil, logging.Errorf("failed to initialize the certificate manager: %v", err)
+	}
+	if certDuration < time.Hour {
+		// the default value for CertCallbackRefreshDuration (5min) is too long for short-lived certs,
+		// set it to a more sensible value
+		transport.CertCallbackRefreshDuration = time.Second * 10
+	}
+	certManager.Start()
+
+	logging.Verbosef("Waiting for certificate")
+	var storeErr error
+	err = wait.PollWithContext(context.TODO(), time.Second, 2*time.Minute, func(_ context.Context) (bool, error) {
+		var currentCert *tls.Certificate
+		currentCert, storeErr = certificateStore.Current()
+		return currentCert != nil && storeErr == nil, nil
+	})
+	if err != nil {
+		return nil, logging.Errorf("certificate was not signed, last cert store err: %v err: %v", storeErr, err)
+	}
+	logging.Verbosef("Certificate found!")
+
+	return newClientInfo(config)
+}
+
+// InClusterK8sClient returns the `k8s.ClientInfo` struct to use to connect to
+// the k8s API.
+func InClusterK8sClient() (*ClientInfo, error) {
+	clientInfo, err := GetK8sClient("", nil)
+	if err != nil {
+		return nil, err
+	}
+	if clientInfo == nil {
+		return nil, fmt.Errorf("failed to create in-cluster kube client")
+	}
+	return clientInfo, err
+}
+
+// SetK8sClientInformers adds informer structure to ClientInfo to utilize in thick daemon
+func (c *ClientInfo) SetK8sClientInformers(podInformer, netDefInformer cache.SharedIndexInformer) {
+	c.PodInformer = podInformer
+	c.NetDefInformer = netDefInformer
+}
+
+// GetK8sClient gets client info from kubeconfig
+func GetK8sClient(kubeconfig string, kubeClient *ClientInfo) (*ClientInfo, error) {
+	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
+	// If we get a valid kubeClient (eg from testcases) just return that
+	// one.
+	if kubeClient != nil {
+		return kubeClient, nil
+	}
+
+	var err error
+	var config *rest.Config
+
+	// Otherwise try to create a kubeClient from a given kubeConfig
+	if kubeconfig != "" {
+		// uses the current context in kubeconfig
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v: %v", kubeconfig, err)
+		}
+	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
+		// Try in-cluster config where multus might be running in a kubernetes pod
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, logging.Errorf("GetK8sClient: failed to get context for in-cluster kube config: %v", err)
+		}
+	} else {
+		// No kubernetes config; assume we shouldn't talk to Kube at all
+		return nil, nil
+	}
+
+	// Specify that we use gRPC
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+	// Set the config timeout to one minute.
+	config.Timeout = time.Minute
+	// Allow multus (especially in server mode) to make more concurrent requests
+	// to reduce client-side throttling
+	config.QPS = 50
+	config.Burst = 50
+
+	return newClientInfo(config)
+}
+
+// newClientInfo returns a `ClientInfo` from a configuration created from an
+// existing kubeconfig file.
+func newClientInfo(config *rest.Config) (*ClientInfo, error) {
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	netclient, err := netclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "multus"})
+	return &ClientInfo{
+		Client:           client,
+		NetClient:        netclient,
+		EventBroadcaster: broadcaster,
+		EventRecorder:    recorder,
+	}, nil
+}

pkg/kubeletclient/kubeletclient.go

@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"golang.org/x/net/context"
@@ -137,19 +138,45 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou
 	for _, pr := range rc.resources {
 		if pr.Name == name && pr.Namespace == ns {
 			for _, cnt := range pr.Containers {
-				for _, dev := range cnt.Devices {
-					if rInfo, ok := resourceMap[dev.ResourceName]; ok {
-						rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...)
-					} else {
-						resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds}
-					}
-				}
+				rc.getDevicePluginResources(cnt.Devices, resourceMap)
+				rc.getDRAResources(cnt.DynamicResources, resourceMap)
 			}
 		}
 	}
 	return resourceMap, nil
 }
 
+func (rc *kubeletClient) getDevicePluginResources(devices []*podresourcesapi.ContainerDevices, resourceMap map[string]*types.ResourceInfo) {
+	for _, dev := range devices {
+		if rInfo, ok := resourceMap[dev.ResourceName]; ok {
+			rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...)
+		} else {
+			resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds}
+		}
+	}
+}
+
+func (rc *kubeletClient) getDRAResources(dynamicResources []*podresourcesapi.DynamicResource, resourceMap map[string]*types.ResourceInfo) {
+	for _, dynamicResource := range dynamicResources {
+		var deviceIDs []string
+		for _, claimResource := range dynamicResource.ClaimResources {
+			for _, cdiDevice := range claimResource.CdiDevices {
+				res := strings.Split(cdiDevice.Name, "=")
+				if len(res) == 2 {
+					deviceIDs = append(deviceIDs, res[1])
+				} else {
+					logging.Errorf("GetPodResourceMap: Invalid CDI format")
+				}
+			}
+		}
+		if rInfo, ok := resourceMap[dynamicResource.ClaimName]; ok {
+			rInfo.DeviceIDs = append(rInfo.DeviceIDs, deviceIDs...)
+		} else {
+			resourceMap[dynamicResource.ClaimName] = &types.ResourceInfo{DeviceIDs: deviceIDs}
+		}
+	}
+}
+
 func hasKubeletAPIEndpoint(url *url.URL) bool {
 	// Check for kubelet resource API socket file
 	if _, err := os.Stat(url.Path); err != nil {

pkg/kubeletclient/kubeletclient_test.go

@@ -15,6 +15,8 @@
 
 package kubeletclient
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -45,6 +47,7 @@ var (
 
 type fakeResourceServer struct {
 	server *grpc.Server
+	podresourcesapi.UnimplementedPodResourcesListerServer
 }
 
 // TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
@@ -52,11 +55,12 @@ func (m *fakeResourceServer) GetAllocatableResources(_ context.Context, _ *podre
 	return &podresourcesapi.AllocatableResourcesResponse{}, nil
 }
 
-func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
-	podName := "pod-name"
-	podNamespace := "pod-namespace"
-	containerName := "container-name"
+// TODO: This is stub code for test, but we may need to change for the testing we use this API in the future...
+func (m *fakeResourceServer) Get(_ context.Context, _ *podresourcesapi.GetPodResourcesRequest) (*podresourcesapi.GetPodResourcesResponse, error) {
+	return &podresourcesapi.GetPodResourcesResponse{}, nil
+}
 
+func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
 	devs := []*podresourcesapi.ContainerDevices{
 		{
 			ResourceName: "resource",
@@ -64,18 +68,54 @@ func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodR
 		},
 	}
 
+	cdiDevices := []*podresourcesapi.CDIDevice{
+		{
+			Name: "cdi-kind=cdi-resource",
+		},
+	}
+	draDriverName := "dra.example.com"
+	poolName := "worker-1-pool"
+	deviceName := "gpu-1"
+
+	claimsResource := []*podresourcesapi.ClaimResource{
+		{
+			CdiDevices: cdiDevices,
+			DriverName: draDriverName,
+			PoolName: poolName,
+			DeviceName: deviceName,
+		},
+	}
+
+	dynamicResources := []*podresourcesapi.DynamicResource{
+		{
+			ClaimName:      "resource-claim",
+			ClaimNamespace: "dynamic-resource-pod-namespace",
+			ClaimResources: claimsResource,
+		},
+	}
+
 	resp := &podresourcesapi.ListPodResourcesResponse{
 		PodResources: []*podresourcesapi.PodResources{
 			{
-				Name:      podName,
-				Namespace: podNamespace,
+				Name:      "pod-name",
+				Namespace: "pod-namespace",
 				Containers: []*podresourcesapi.ContainerResources{
 					{
-						Name:    containerName,
+						Name:    "container-name",
 						Devices: devs,
 					},
 				},
 			},
+			{
+				Name:      "dynamic-resource-pod-name",
+				Namespace: "dynamic-resource-pod-namespace",
+				Containers: []*podresourcesapi.ContainerResources{
+					{
+						Name:             "dynamic-resource-container-name",
+						DynamicResources: dynamicResources,
+					},
+				},
+			},
 		},
 	}
 	return resp, nil
@@ -181,7 +221,7 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 		})
 	})
 	Context("GetPodResourceMap() with valid pod name and namespace", func() {
-		It("should return no error", func() {
+		It("should return no error with device plugin resource", func() {
 			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
 			fakePod := &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
@@ -209,6 +249,34 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() {
 			Expect(resourceMap).To(Equal(outputRMap))
 		})
 
+		It("should return no error with dynamic resource", func() {
+			podUID := k8sTypes.UID("9f94e27b-4233-43d6-bd10-f73b4de6f456")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "dynamic-resource-pod-name",
+					Namespace: "dynamic-resource-pod-namespace",
+					UID:       podUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "dynamic-resource-container-name",
+						},
+					},
+				},
+			}
+			client, err := getKubeletClient(testKubeletSocket)
+			Expect(err).NotTo(HaveOccurred())
+
+			outputRMap := map[string]*mtypes.ResourceInfo{
+				"resource-claim": {DeviceIDs: []string{"cdi-resource"}},
+			}
+			resourceMap, err := client.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(resourceMap).ShouldNot(BeNil())
+			Expect(resourceMap).To(Equal(outputRMap))
+		})
+
 		It("should return an error with garbage socket value", func() {
 			u, err := url.Parse("/badfilepath!?//")
 			Expect(err).NotTo(HaveOccurred())

pkg/logging/logging.go

@@ -56,25 +56,35 @@ type LogOptions struct {
 
 // SetLogOptions set the LoggingOptions of NetConf
 func SetLogOptions(options *LogOptions) {
+	// logger is used only if filname is supplied
+	if logger == nil || logger.Filename == "" {
+		return
+	}
+
 	// give some default value
-	logger.MaxSize = 100
-	logger.MaxAge = 5
-	logger.MaxBackups = 5
-	logger.Compress = true
+	updatedLogger := lumberjack.Logger{
+		Filename:   logger.Filename,
+		MaxAge:     5,
+		MaxBackups: 5,
+		Compress:   true,
+		MaxSize:    100,
+		LocalTime:  logger.LocalTime,
+	}
 	if options != nil {
 		if options.MaxAge != nil {
-			logger.MaxAge = *options.MaxAge
+			updatedLogger.MaxAge = *options.MaxAge
 		}
 		if options.MaxSize != nil {
-			logger.MaxSize = *options.MaxSize
+			updatedLogger.MaxSize = *options.MaxSize
 		}
 		if options.MaxBackups != nil {
-			logger.MaxBackups = *options.MaxBackups
+			updatedLogger.MaxBackups = *options.MaxBackups
 		}
 		if options.Compress != nil {
-			logger.Compress = *options.Compress
+			updatedLogger.Compress = *options.Compress
 		}
 	}
+	logger = &updatedLogger
 	loggingW = logger
 }
 
@@ -171,18 +181,32 @@ func SetLogStderr(enable bool) {
 
 // SetLogFile sets logging file
 func SetLogFile(filename string) {
+	// logger is used only if filname is supplied
 	if filename == "" {
 		return
 	}
 
-	logger.Filename = filename
-	loggingW = logger
+	updatedLogger := lumberjack.Logger{
+		Filename:   filename,
+		MaxAge:     5,
+		MaxBackups: 5,
+		Compress:   true,
+		MaxSize:    100,
+	}
 
+	if logger != nil {
+		updatedLogger.MaxAge = logger.MaxAge
+		updatedLogger.MaxBackups = logger.MaxBackups
+		updatedLogger.Compress = logger.Compress
+		updatedLogger.MaxSize = logger.MaxSize
+	}
+	logger = &updatedLogger
+	loggingW = logger
 }
 
 func init() {
 	loggingStderr = true
 	loggingW = nil
 	loggingLevel = PanicLevel
-	logger = &lumberjack.Logger{}
+	logger = nil
 }

pkg/logging/logging_test.go

@@ -15,6 +15,8 @@
 
 package logging
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"fmt"
 	"os"
@@ -106,13 +108,13 @@ var _ = Describe("logging operations", func() {
 		Verbosef("foobar")
 		Expect(Errorf("foobar")).NotTo(BeNil())
 		Panicf("foobar")
-		logger.Filename = ""
+		logger = nil
 		loggingW = nil
 		err = os.RemoveAll(tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		// Revert the log variable to init
 		loggingW = nil
-		logger = &lumberjack.Logger{}
+		logger = nil
 	})
 
 	// Tests public getter

pkg/multus/multus.go

@@ -18,6 +18,7 @@ package multus
 import (
 	"context"
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"net"
 	"os"
@@ -30,6 +31,7 @@ import (
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
@@ -46,21 +48,18 @@ import (
 )
 
 const (
-	shortPollDuration = 250 * time.Millisecond
-	shortPollTimeout  = 2500 * time.Millisecond
+	shortPollDuration    = 250 * time.Millisecond
+	informerPollDuration = 50 * time.Millisecond
+	shortPollTimeout     = 2500 * time.Millisecond
 )
 
 var (
-	version       = "master@git"
-	commit        = "unknown commit"
-	date          = "unknown date"
-	gitTreeState  = ""
-	releaseStatus = ""
-)
-
-var (
-	pollDuration = 1000 * time.Millisecond
-	pollTimeout  = 45 * time.Second
+	version        = "master@git"
+	commit         = "unknown commit"
+	date           = "unknown date"
+	gitTreeState   = ""
+	releaseStatus  = ""
+	errPodNotFound = fmt.Errorf("pod not found during Multus GetPod")
 )
 
 // PrintVersionString ...
@@ -135,15 +134,49 @@ func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetCo
 	return err
 }
 
-func deleteDelegates(containerID, dataDir string) error {
-	logging.Debugf("deleteDelegates: %s, %s", containerID, dataDir)
-
-	path := filepath.Join(dataDir, containerID)
-	if err := os.Remove(path); err != nil {
-		return logging.Errorf("deleteDelegates: error in deleting the delegates : %v", err)
+func getValidAttachmentFromCache(b []byte) (string, string, error) {
+	type simpleCacheV1 struct {
+		Kind        string `json:"kind"`
+		ContainerID string `json:"containerId"`
+		IfName      string `json:"ifName"`
 	}
 
-	return nil
+	cache := &simpleCacheV1{}
+	if err := json.Unmarshal(b, cache); err != nil {
+		return "", "", fmt.Errorf("getValidAttachmentFromCache: invalid json: %v", err)
+	}
+
+	if cache.ContainerID == "" || cache.IfName == "" {
+		return "", "", fmt.Errorf("invalid cache: containerID:%q, ifName:%q", cache.ContainerID, cache.IfName)
+	}
+
+	return cache.ContainerID, cache.IfName, nil
+}
+
+func gatherValidAttachmentsFromCache(cniDir string) ([]cnitypes.GCAttachment, error) {
+	cacheDir := filepath.Join(cniDir, "results")
+	dirEntries, err := os.ReadDir(cacheDir)
+	if err != nil {
+		return nil, err
+	}
+
+	allAttachments := []cnitypes.GCAttachment{}
+	for _, dirEnt := range dirEntries {
+		path := filepath.Join(cacheDir, dirEnt.Name())
+		delegatesBytes, err := os.ReadFile(path)
+		// if delegates cannot read that, skipped for now (because cannot recover).
+		if err != nil {
+			logging.Errorf("gatherSavedDelegates: cannot read %q, skipped to add", path)
+			continue
+		}
+		containerID, ifName, err := getValidAttachmentFromCache(delegatesBytes)
+		if err != nil {
+			logging.Errorf("gatherSavedDelegates: cannot read cache, skipped to add: %v", err)
+			continue
+		}
+		allAttachments = append(allAttachments, cnitypes.GCAttachment{ContainerID: containerID, IfName: ifName})
+	}
+	return allAttachments, nil
 }
 
 func validateIfName(nsname string, ifname string) error {
@@ -227,16 +260,61 @@ func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.Net
 	return err
 }
 
-func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
+func confStatus(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("confStatus: %v, %s", rt, string(rawNetconf))
+
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
+
+	conf, err := libcni.ConfFromBytes(rawNetconf)
+	if err != nil {
+		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	}
+
+	if gt, _ := cniversion.GreaterThanOrEqualTo(conf.Network.CNIVersion, "1.1.0"); !gt {
+		logging.Debugf("confStatus: skipping STATUS for network %q type %q cniVersion %q (< 1.1.0)",
+			conf.Network.Name, conf.Network.Type, conf.Network.CNIVersion)
+		return nil
+	}
+
+	confList := &libcni.NetworkConfigList{
+		Name:       conf.Network.Name,
+		CNIVersion: conf.Network.CNIVersion,
+		Plugins:    []*libcni.PluginConfig{conf},
+	}
+
+	err = cniNet.GetStatusNetworkList(context.Background(), confList)
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		return logging.Errorf("error in getting result from StatusNetworkList: %v", err)
+	}
+
+	return err
+}
+
+func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, cniConfList *libcni.NetworkConfigList, multusNetconf *types.NetConf, exec invoke.Exec) (cnitypes.Result, error) {
 	logging.Debugf("conflistAdd: %v, %s", rt, string(rawnetconflist))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
 	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
 	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
 
-	confList, err := libcni.ConfListFromBytes(rawnetconflist)
-	if err != nil {
-		return nil, logging.Errorf("conflistAdd: error converting the raw bytes into a conflist: %v", err)
+	var confList *libcni.NetworkConfigList
+	var err error
+
+	// This may wind up being set during parsing the default network config.
+	// In this case -- we'll use it as passed. Otherwise, we'll recalculate it.
+	if len(cniConfList.Plugins) > 0 {
+		confList = cniConfList
+	} else {
+		confList, err = libcni.NetworkConfFromBytes(rawnetconflist)
+		if err != nil {
+			return nil, logging.Errorf("conflistAdd: error converting the raw bytes into a conflist: %v", err)
+		}
 	}
 
 	result, err := cniNet.AddNetworkList(context.Background(), confList, rt)
@@ -287,6 +365,33 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *t
 	return err
 }
 
+func conflistStatus(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
+	logging.Debugf("conflistStatus: %v, %s", rt, string(rawnetconflist))
+
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, multusNetconf.CNIDir, exec)
+
+	confList, err := libcni.ConfListFromBytes(rawnetconflist)
+	if err != nil {
+		return logging.Errorf("conflistStatus: error converting the raw bytes into a conflist: %v", err)
+	}
+	if gt, _ := cniversion.GreaterThanOrEqualTo(confList.CNIVersion, "1.1.0"); !gt {
+		logging.Debugf("conflistStatus: skipping STATUS for network list %q cniVersion %q (< 1.1.0)", confList.Name, confList.CNIVersion)
+	}
+
+	err = cniNet.GetStatusNetworkList(context.Background(), confList)
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		return logging.Errorf("conflistStatus: error in getting result from StatusNetworkList: %v", err)
+	}
+
+	return err
+}
+
 // DelegateAdd ...
 func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
 	logging.Debugf("DelegateAdd: %v, %v, %v", exec, delegate, rt)
@@ -330,7 +435,8 @@ func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 	var result cnitypes.Result
 	var err error
 	if delegate.ConfListPlugin {
-		result, err = conflistAdd(rt, delegate.Bytes, multusNetconf, exec)
+		// TODO: why are we passing bytes here? don't we have a better representation of it?
+		result, err = conflistAdd(rt, delegate.Bytes, &delegate.CNINetworkConfigList, multusNetconf, exec)
 		if err != nil {
 			return nil, err
 		}
@@ -369,11 +475,14 @@ func DelegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, dele
 	}
 
 	if pod != nil {
-		// send kubernetes events
-		if delegate.Name != "" {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
-		} else {
-			kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+		// check Interfaces and IPs because some CNI plugin just return empty result
+		if res.Interfaces != nil || res.IPs != nil {
+			// send kubernetes events
+			if delegate.Name != "" {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v from %s", rt.IfName, ips, delegate.Name)
+			} else {
+				kubeClient.Eventf(pod, v1.EventTypeNormal, "AddedInterface", "Add %s %v", rt.IfName, ips)
+			}
 		}
 	} else {
 		// for further debug https://github.com/k8snetworkplumbingwg/multus-cni/issues/481
@@ -412,6 +521,46 @@ func DelegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *li
 	return err
 }
 
+// DelegateStatus ...
+func DelegateStatus(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+	logging.Debugf("DelegateStatus: %v, %v, %v", exec, delegateConf, rt)
+
+	isConfList := delegateConf.ConfListPlugin
+	if !isConfList && delegateConf.Conf.Type == "" && delegateConf.ConfList.Name != "" {
+		isConfList = true
+	}
+
+	if logging.GetLoggingLevel() >= logging.VerboseLevel {
+		var cniConfName string
+		if isConfList {
+			cniConfName = delegateConf.ConfList.Name
+		} else {
+			cniConfName = delegateConf.Conf.Name
+		}
+		logging.Verbosef("Status: %s:%s:%s(%s):%s %s", rt.Args[1][1], rt.Args[2][1], delegateConf.Name, cniConfName, rt.IfName, string(delegateConf.Bytes))
+	}
+
+	var err error
+	if isConfList {
+		err = conflistStatus(rt, delegateConf.Bytes, multusNetconf, exec)
+	} else {
+		err = confStatus(rt, delegateConf.Bytes, multusNetconf, exec)
+	}
+
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		if isConfList {
+			return logging.Errorf("DelegateStatus: error invoking ConflistStatus - %q: %v", delegateConf.ConfList.Name, err)
+		}
+		return logging.Errorf("DelegateStatus: error invoking ConfStatus - %q: %v", delegateConf.Conf.Type, err)
+	}
+
+	return err
+}
+
 // DelegateDel ...
 func DelegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
 	logging.Debugf("DelegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
@@ -472,7 +621,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type
 
 	// Check if we had any errors, and send them all back.
 	if len(errorstrings) > 0 {
-		return fmt.Errorf(strings.Join(errorstrings, " / "))
+		return fmt.Errorf("%s", strings.Join(errorstrings, " / "))
 	}
 
 	return nil
@@ -508,7 +657,7 @@ func isCriticalRequestRetriable(err error) bool {
 
 // GetPod retrieves Kubernetes Pod object from given namespace/name in k8sArgs (i.e. cni args)
 // GetPod also get pod UID, but it is not used to retrieve, but it is used for double check
-func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (*v1.Pod, error) {
+func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, isDel bool) (*v1.Pod, error) {
 	if kubeClient == nil {
 		return nil, nil
 	}
@@ -517,31 +666,67 @@ func GetPod(kubeClient *k8s.ClientInfo, k8sArgs *types.K8sArgs, warnOnly bool) (
 	podName := string(k8sArgs.K8S_POD_NAME)
 	podUID := string(k8sArgs.K8S_POD_UID)
 
-	pod, err := kubeClient.GetPod(podNamespace, podName)
-	if err != nil {
-		// in case of a retriable error, retry 10 times with 0.25 sec interval
-		if isCriticalRequestRetriable(err) {
-			waitErr := wait.PollImmediate(shortPollDuration, shortPollTimeout, func() (bool, error) {
-				pod, err = kubeClient.GetPod(podNamespace, podName)
-				return pod != nil, err
-			})
-			// retry failed, then return error with retry out
-			if waitErr != nil {
-				return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
+	// Keep track of how long getting the pod takes
+	logging.Debugf("GetPod for [%s/%s] starting", podNamespace, podName)
+	start := time.Now()
+	defer func() {
+		logging.Debugf("GetPod for [%s/%s] took %v", podNamespace, podName, time.Since(start))
+	}()
+
+	// Use a fairly long 0.25 sec interval so we don't hammer the apiserver
+	pollDuration := shortPollDuration
+	retryOnNotFound := func(error) bool {
+		return false
+	}
+
+	if kubeClient.PodInformer != nil {
+		logging.Debugf("GetPod for [%s/%s] will use informer cache", podNamespace, podName)
+		// Use short retry intervals with the informer since it's a local cache
+		pollDuration = informerPollDuration
+		// Retry NotFound on ADD since the cache may be a bit behind the apiserver
+		retryOnNotFound = func(e error) bool {
+			return !isDel && errors.IsNotFound(e)
+		}
+	}
+
+	var pod *v1.Pod
+	if err := wait.PollImmediate(pollDuration, shortPollTimeout, func() (bool, error) {
+		var getErr error
+		// Use context with a short timeout so the call to API server doesn't take too long.
+		ctx, cancel := context.WithTimeout(context.TODO(), pollDuration)
+		defer cancel()
+		pod, getErr = kubeClient.GetPodContext(ctx, podNamespace, podName)
+		if isCriticalRequestRetriable(getErr) || retryOnNotFound(getErr) {
+			return false, nil
+		}
+		return pod != nil, getErr
+	}); err != nil {
+		if errors.IsNotFound(err) {
+			// When pods are not found, this is "OK", it's a known condition for rapidly deleted pods, we'll just warn on it.
+			if !isDel {
+				logging.Verbosef("Warning: GetPod for [%s/%s] resulted in pod not found during CNI ADD (pod may have already been deleted): %v", podNamespace, podName, err)
 			}
-		} else if warnOnly && errors.IsNotFound(err) {
-			// If not found, proceed to remove interface with cache
-			return nil, nil
-		} else {
-			// Other case, return error
-			return nil, cmdErr(k8sArgs, "error getting pod: %v", err)
+			return nil, errPodNotFound
+		}
+		// Try one more time to get the pod directly from the apiserver;
+		// TODO: figure out why static pods don't show up via the informer
+		// and always hit this case.
+		ctx, cancel := context.WithTimeout(context.TODO(), pollDuration)
+		defer cancel()
+		pod, err = kubeClient.GetPodAPILiveQuery(ctx, podNamespace, podName)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				logging.Verbosef("Warning: On live query retry, [%s/%s] pod not found during CNI ADD (pod may have already been deleted): %v", podNamespace, podName, err)
+				return nil, errPodNotFound
+			}
+			return nil, cmdErr(k8sArgs, "error waiting for pod: %v", err)
 		}
 	}
 
 	// In case of static pod, UID through kube api is different because of mirror pod, hence it is expected.
 	if podUID != "" && string(pod.UID) != podUID && !k8s.IsStaticPod(pod) {
 		msg := fmt.Sprintf("expected pod UID %q but got %q from Kube API", podUID, pod.UID)
-		if warnOnly {
+		if isDel {
 			// On CNI DEL we just operate on the cache when these mismatch, we don't error out.
 			// For example: stateful sets namespace/name can remain the same while podUID changes.
 			logging.Verbosef("warning: %s", msg)
@@ -572,17 +757,18 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 	}
 
 	if n.ReadinessIndicatorFile != "" {
-		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-			_, err := os.Stat(n.ReadinessIndicatorFile)
-			return err == nil, nil
-		})
-		if err != nil {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
 			return nil, cmdErr(k8sArgs, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
 		}
 	}
 
 	pod, err := GetPod(kubeClient, k8sArgs, false)
 	if err != nil {
+		if stderrors.Is(err, errPodNotFound) {
+			emptyResult := emptyCNIResult(args, n.CNIVersion)
+			logging.Verbosef("CmdAdd: Warning: pod [%s/%s] not found, exiting with empty CNI result: %v", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME, emptyResult)
+			return emptyResult, nil
+		}
 		return nil, err
 	}
 
@@ -604,6 +790,36 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		return nil, cmdErr(k8sArgs, "error loading k8s delegates k8s args: %v", err)
 	}
 
+	// we add to the auxiliary CNI chain here.
+	if n.AuxiliaryCNIChainName != "" {
+		logging.Debugf("Using AuxiliaryCNIChainName: %v", n.AuxiliaryCNIChainName)
+
+		// create an passthru cni conflist configuration with our aux chain cni chain name.
+		jsonString := fmt.Sprintf(`{"cniVersion":"%s","name":"%s","plugins":[{"type":"passthru","name":"passthru-cni"}]}`, n.CNIVersion, n.AuxiliaryCNIChainName)
+
+		// Convert the JSON string to a byte array
+		byteArray := []byte(jsonString)
+
+		// Let's try to get the cni path from the ClusterNetwork
+		if !strings.Contains(n.ClusterNetwork, "/") {
+			return nil, cmdErr(k8sArgs, "auxiliary chain used but ClusterNetwork must be a path, and it is not a path: %v", n.ClusterNetwork)
+		}
+
+		// Get the directory part of the ClusterNetwork path
+		// TODO: This could probably be improved.
+		cniPath := filepath.Dir(n.ClusterNetwork)
+
+		// Load chained delegates
+		delegate := k8s.LoadChainedDelegatesFromBytes(byteArray, cniPath)
+		if delegate != nil {
+			// Only if additional plugins were listed do we add this aux chain delegate.
+			if len(delegate.ConfList.Plugins) > 1 {
+				// Add the resulting delegate to n.Delegates
+				n.Delegates = append(n.Delegates, delegate)
+			}
+		}
+	}
+
 	// cache the multus config
 	if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
 		return nil, cmdErr(k8sArgs, "error saving the delegates: %v", err)
@@ -636,71 +852,80 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			return nil, cmdPluginErr(k8sArgs, netName, "error adding container to network %q: %v", netName, err)
 		}
 
-		// Remove gateway from routing table if the gateway is not used
-		deleteV4gateway := false
-		deleteV6gateway := false
-		adddefaultgateway := false
-		if delegate.IsFilterV4Gateway {
-			deleteV4gateway = true
-			logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
-		} else {
-			// Otherwise, determine if this interface now gets our default route.
-			// According to
-			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
-			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
-			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
-				deleteV4gateway = true
-				adddefaultgateway = true
-				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
-			}
-		}
-
-		if delegate.IsFilterV6Gateway {
-			deleteV6gateway = true
-			logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
-		} else {
-			// Otherwise, determine if this interface now gets our default route.
-			// According to
-			// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
-			// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
-			// else we'll update the defaultgateway to the one specified.
-			if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
-				deleteV6gateway = true
-				adddefaultgateway = true
-				logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
-			}
-		}
-
-		// Remove gateway if `default-route` network selection is specified
-		if deleteV4gateway || deleteV6gateway {
-			err = netutils.DeleteDefaultGW(args.Netns, ifName)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
-			}
-			err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
-			}
-		}
-
-		// Here we'll set the default gateway which specified in `default-route` network selection
-		if adddefaultgateway {
-			err = netutils.SetDefaultGW(args.Netns, ifName, *delegate.GatewayRequest)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
-			}
-			err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
-			if err != nil {
-				return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
-			}
-		}
-
 		// Master plugin result is always used if present
 		if delegate.MasterPlugin || result == nil {
 			result = tmpResult
 		}
 
+		res, err := cni100.NewResultFromResult(tmpResult)
+		if err != nil {
+			logging.Errorf("CmdAdd: failed to read result: %v, but proceed", err)
+		}
+
+		// check Interfaces and IPs because some CNI plugin does not create any interface
+		// and just returns empty result
+		if res != nil && (res.Interfaces != nil || res.IPs != nil) {
+			// Remove gateway from routing table if the gateway is not used
+			deleteV4gateway := false
+			deleteV6gateway := false
+			adddefaultgateway := false
+			if delegate.IsFilterV4Gateway {
+				deleteV4gateway = true
+				logging.Debugf("Marked interface %v for v4 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV4gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			if delegate.IsFilterV6Gateway {
+				deleteV6gateway = true
+				logging.Debugf("Marked interface %v for v6 gateway deletion", ifName)
+			} else {
+				// Otherwise, determine if this interface now gets our default route.
+				// According to
+				// https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ (4.1.2.1.9)
+				// the list can be empty; if it is, we'll assume the CNI's config for the default gateway holds,
+				// else we'll update the defaultgateway to the one specified.
+				if delegate.GatewayRequest != nil && len(*delegate.GatewayRequest) != 0 {
+					deleteV6gateway = true
+					adddefaultgateway = true
+					logging.Debugf("Detected gateway override on interface %v to %v", ifName, delegate.GatewayRequest)
+				}
+			}
+
+			// Remove gateway if `default-route` network selection is specified
+			if deleteV4gateway || deleteV6gateway {
+				err = netutils.DeleteDefaultGW(args.Netns, ifName)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
+				}
+				err = netutils.DeleteDefaultGWCache(n.CNIDir, rt, netName, ifName, deleteV4gateway, deleteV6gateway)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error deleting default gateway in cache: %v", err)
+				}
+			}
+
+			// Here we'll set the default gateway which specified in `default-route` network selection
+			if adddefaultgateway {
+				err = netutils.SetDefaultGW(args.Netns, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
+				}
+				err = netutils.AddDefaultGWCache(n.CNIDir, rt, netName, ifName, *delegate.GatewayRequest)
+				if err != nil {
+					return nil, cmdErr(k8sArgs, "error setting default gateway in cache: %v", err)
+				}
+			}
+		}
+
 		// Read devInfo from CNIDeviceInfoFile if it exists so
 		// it can be copied to the NetworkStatus.
 		devinfo, err := getDelegateDeviceInfo(delegate, rt)
@@ -710,15 +935,18 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 			logging.Debugf("CmdAdd: getDelegateDeviceInfo returned an error - err=%v", err)
 		}
 
-		// create the network status, only in case Multus as kubeconfig
+		// Create the network statuses, only in case Multus has kubeconfig
 		if kubeClient != nil && kc != nil {
 			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
-				delegateNetStatus, err := nadutils.CreateNetworkStatus(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
+				delegateNetStatuses, err := nadutils.CreateNetworkStatuses(tmpResult, delegate.Name, delegate.MasterPlugin, devinfo)
 				if err != nil {
-					return nil, cmdErr(k8sArgs, "error setting network status: %v", err)
+					return nil, cmdErr(k8sArgs, "error setting network statuses: %v", err)
 				}
 
-				netStatus = append(netStatus, *delegateNetStatus)
+				// Append all returned statuses after dereferencing each
+				for _, status := range delegateNetStatuses {
+					netStatus = append(netStatus, *status)
+				}
 			}
 		} else if devinfo != nil {
 			// Warn that devinfo exists but could not add it to downwards API
@@ -726,15 +954,17 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 	}
 
-	// set the network status annotation in apiserver, only in case Multus as kubeconfig
+	// set the network status annotation in apiserver, only in case Multus has kubeconfig
 	if kubeClient != nil && kc != nil {
 		if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAME), n.SystemNamespaces) {
 			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
-				if strings.Contains(err.Error(), "failed to query the pod") {
-					return nil, cmdErr(k8sArgs, "error setting the networks status, pod was already deleted: %v", err)
+				if strings.Contains(err.Error(), `pod "`) && strings.Contains(err.Error(), `" not found`) {
+					// Tolerate issues with writing the status due to pod deletion, and log them.
+					logging.Verbosef("warning: tolerated failure writing network status (pod not found): %v", err)
+				} else {
+					return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 				}
-				return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 			}
 		}
 	}
@@ -776,21 +1006,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		return err
 	}
 
-	skipStatusUpdate := false
 	netns, err := ns.GetNS(args.Netns)
-	if err != nil {
-		// if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
-		// so don't return an error if the device is already removed.
-		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
-		_, ok := err.(ns.NSPathNotExistErr)
-		skipStatusUpdate = true
-		if ok {
-			logging.Debugf("CmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
-		} else {
-			logging.Debugf("CmdDel: WARNING failed to open netns %q: %v", netns, err)
-		}
-	}
-
 	if netns != nil {
 		defer netns.Close()
 	}
@@ -801,12 +1017,13 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 	}
 
 	if in.ReadinessIndicatorFile != "" {
-		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
-			_, err := os.Stat(in.ReadinessIndicatorFile)
-			return err == nil, nil
-		})
+		readinessfileexists, err := types.ReadinessIndicatorExistsNow(in.ReadinessIndicatorFile)
 		if err != nil {
-			return cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile (on del): %v", err)
+			return cmdErr(k8sArgs, "error checking readinessindicatorfile on CNI DEL @ %v: %v", in.ReadinessIndicatorFile, err)
+		}
+
+		if !readinessfileexists {
+			logging.Verbosef("warning: readinessindicatorfile @ %v does not exist on CNI DEL", in.ReadinessIndicatorFile)
 		}
 	}
 
@@ -819,8 +1036,6 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 	if err != nil {
 		// GetPod may be failed but just do print error in its log and continue to delete
 		logging.Errorf("Multus: GetPod failed: %v, but continue to delete", err)
-		// skip status update because k8s api seems to be stucked
-		skipStatusUpdate = true
 	}
 
 	// Read the cache to get delegates json for the pod
@@ -885,21 +1100,6 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
-	if kubeClient != nil {
-		if !skipStatusUpdate {
-			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
-				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
-				if err != nil {
-					// error happen but continue to delete
-					logging.Errorf("Multus: error unsetting the networks status: %v", err)
-				}
-			}
-		} else {
-			logging.Debugf("WARNING: Unset SetNetworkStatus skipped")
-		}
-	}
-
 	e := delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
 
 	// Enable Option only delegate plugin delete success to delete cache file
@@ -924,3 +1124,153 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 
 	return e
 }
+
+// CmdStatus ...
+func CmdStatus(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	n, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("CmdStatus: %v, %v, %v", args, exec, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error loading netconf: %v", err)
+	}
+
+	kubeClient, err = k8s.GetK8sClient(n.Kubeconfig, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error getting k8s client: %v", err)
+	}
+
+	if n.ReadinessIndicatorFile != "" {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
+			return cmdErr(nil, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
+		}
+	}
+
+	if n.ClusterNetwork != "" {
+		_, err = k8s.GetDefaultNetworks(nil, n, kubeClient, nil)
+		if err != nil {
+			return cmdErr(nil, "failed to get clusterNetwork: %v", err)
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	// invoke delegate's STATUS command
+	// we only need to check cluster network status
+	delegate := n.Delegates[0]
+	if !delegate.ConfListPlugin {
+		return confStatus(&libcni.RuntimeConf{}, delegate.Bytes, n, exec)
+	}
+
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{n.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
+
+	conf, err := libcni.ConfListFromBytes(delegate.Bytes)
+	if err != nil {
+		return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+	}
+
+	err = cniNet.GetStatusNetworkList(context.Background(), conf)
+	if err != nil {
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return err
+		}
+		return logging.Errorf("error in STATUS command: %v", err)
+	}
+
+	return nil
+}
+
+// CmdGC ...
+func CmdGC(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+	n, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("CmdStatus: %v, %v, %v", args, exec, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error loading netconf: %v", err)
+	}
+
+	kubeClient, err = k8s.GetK8sClient(n.Kubeconfig, kubeClient)
+	if err != nil {
+		return cmdErr(nil, "error getting k8s client: %v", err)
+	}
+
+	if n.ReadinessIndicatorFile != "" {
+		if err := types.GetReadinessIndicatorFile(n.ReadinessIndicatorFile); err != nil {
+			return cmdErr(nil, "have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", n.ReadinessIndicatorFile, err)
+		}
+	}
+
+	if n.ClusterNetwork != "" {
+		_, err = k8s.GetDefaultNetworks(nil, n, kubeClient, nil)
+		if err != nil {
+			return cmdErr(nil, "failed to get clusterNetwork: %v", err)
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	// invoke delegate's GC command
+	// we only need to check cluster network status
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append([]string{n.BinDir}, binDirs...)
+	cniNet := libcni.NewCNIConfigWithCacheDir(binDirs, n.CNIDir, exec)
+
+	delegate := n.Delegates[0]
+	isConfList := delegate.ConfListPlugin
+	if !isConfList && delegate.Conf.Type == "" && delegate.ConfList.Name != "" {
+		isConfList = true
+	}
+
+	var confList *libcni.NetworkConfigList
+	if isConfList {
+		confList, err = libcni.ConfListFromBytes(delegate.Bytes)
+		if err != nil {
+			return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+		}
+	} else {
+		conf, err := libcni.ConfFromBytes(delegate.Bytes)
+		if err != nil {
+			return logging.Errorf("error in converting the raw bytes to conf: %v", err)
+		}
+		confList = &libcni.NetworkConfigList{
+			Name:       conf.Network.Name,
+			CNIVersion: conf.Network.CNIVersion,
+			Plugins:    []*libcni.PluginConfig{conf},
+		}
+	}
+
+	validAttachments, err := gatherValidAttachmentsFromCache(n.CNIDir)
+	if err != nil {
+		return logging.Errorf("error in gather valid attachments: %v", err)
+	}
+
+	err = cniNet.GCNetworkList(context.TODO(), confList, &libcni.GCArgs{
+		ValidAttachments: validAttachments,
+	})
+	if err != nil {
+		return logging.Errorf("error in GC command: %v", err)
+	}
+
+	return nil
+}
+
+func emptyCNIResult(args *skel.CmdArgs, cniVersion string) *cni100.Result {
+	return &cni100.Result{
+		CNIVersion: cniVersion,
+		Interfaces: []*cni100.Interface{
+			{
+				Name:    args.IfName,
+				Sandbox: args.Netns,
+			},
+		},
+		IPs: []*cni100.IPConfig{
+			{
+				Address: net.IPNet{
+					IP:   net.ParseIP("0.0.0.0"),
+					Mask: net.CIDRMask(0, 32),
+				},
+				Gateway: net.ParseIP("0.0.0.0"),
+			},
+		},
+	}
+}

pkg/multus/multus_cni020_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -24,10 +26,8 @@ import (
 	types020 "github.com/containernetworking/cni/pkg/types/020"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 
@@ -41,20 +41,6 @@ var _ = Describe("multus operations", func() {
 		err := saveScratchNetConf("123456789", "", meme)
 		Expect(err).To(HaveOccurred())
 	})
-
-	It("fails to delete delegates with bad filepath", func() {
-		err := deleteDelegates("123456789", "bad!file!~?Path$^")
-		Expect(err).To(HaveOccurred())
-	})
-
-	It("delete delegates given good filepath", func() {
-		os.MkdirAll("/opt/cni/bin", 0755)
-		d1 := []byte("blah")
-		os.WriteFile("/opt/cni/bin/123456789", d1, 0644)
-
-		err := deleteDelegates("123456789", "/opt/cni/bin")
-		Expect(err).NotTo(HaveOccurred())
-	})
 })
 
 var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
@@ -99,7 +85,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -154,6 +140,67 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.2.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.2.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.2.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &types020.Result{
+			CNIVersion: "0.2.0",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.2.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("executes delegates given faulty namespace", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -162,7 +209,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -216,7 +263,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -283,7 +330,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -329,7 +376,7 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -706,66 +753,4 @@ var _ = Describe("multus operations cniVersion 0.2.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
-		args := &skel.CmdArgs{
-			ContainerID: "123456789",
-			Netns:       testNS.Path(),
-			IfName:      "eth0",
-			StdinData: []byte(`{
-	    "name": "node-cni-network",
-	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
-	    "defaultnetworkwaitseconds": 3,
-	    "delegates": [{
-	        "name": "weave1",
-	        "cniVersion": "0.2.0",
-	        "type": "weave-net"
-	    },{
-	        "name": "other1",
-	        "cniVersion": "0.2.0",
-	        "type": "other-plugin"
-	    }]
-	}`),
-		}
-
-		fExec := newFakeExec()
-		expectedResult1 := &types020.Result{
-			CNIVersion: "0.2.0",
-			IP4: &types020.IPConfig{
-				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-		}
-		expectedConf1 := `{
-	    "name": "weave1",
-	    "cniVersion": "0.2.0",
-	    "type": "weave-net"
-	}`
-		fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil)
-
-		expectedResult2 := &types020.Result{
-			CNIVersion: "0.2.0",
-			IP4: &types020.IPConfig{
-				IP: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "0.2.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil)
-
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"0.2.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
-
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
-		Expect(err).To(HaveOccurred())
-	})
-
 })

pkg/multus/multus_cni040_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -24,10 +26,8 @@ import (
 	cni040 "github.com/containernetworking/cni/pkg/types/040"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -78,7 +78,7 @@ var _ = Describe("multus operations cniVersion 0.3.1 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -247,6 +247,67 @@ var _ = Describe("multus operations cniVersion 0.3.1 config", func() {
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "0.3.1",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "0.3.1",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni040.Result{
+			CNIVersion: "0.3.1",
+			IPs: []*cni040.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			}},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "0.3.1",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin040(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &cni040.Result{
+			CNIVersion: "0.3.1",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "0.3.1",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin040(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		r := result.(*cni040.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("fails when pod UID is provided and does not match Kube API pod UID", func() {
 		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
 		net1 := `{
@@ -590,7 +651,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -658,7 +719,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -714,7 +775,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -790,7 +851,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -859,7 +920,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -905,7 +966,7 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -1439,67 +1500,4 @@ var _ = Describe("multus operations cniVersion 0.4.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
-		args := &skel.CmdArgs{
-			ContainerID: "123456789",
-			Netns:       testNS.Path(),
-			IfName:      "eth0",
-			StdinData: []byte(`{
-	    "name": "node-cni-network",
-	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
-	    "defaultnetworkwaitseconds": 3,
-	    "delegates": [{
-	        "name": "weave1",
-	        "cniVersion": "0.4.0",
-	        "type": "weave-net"
-	    },{
-	        "name": "other1",
-	        "cniVersion": "0.4.0",
-	        "type": "other-plugin"
-	    }]
-	}`),
-		}
-
-		fExec := newFakeExec()
-		expectedResult1 := &cni040.Result{
-			CNIVersion: "0.4.0",
-			IPs: []*cni040.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-			},
-		}
-		expectedConf1 := `{
-	    "name": "weave1",
-	    "cniVersion": "0.4.0",
-	    "type": "weave-net"
-	}`
-		fExec.addPlugin040(nil, "eth0", expectedConf1, expectedResult1, nil)
-
-		expectedResult2 := &cni040.Result{
-			CNIVersion: "0.4.0",
-			IPs: []*cni040.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-			},
-		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "0.4.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin040(nil, "net1", expectedConf2, expectedResult2, nil)
-
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"0.4.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
-
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
-		Expect(err).To(HaveOccurred())
-	})
 })

pkg/multus/multus_cni100_test.go

@@ -14,29 +14,94 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
+	"sync"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	testhelpers "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/testing"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	kapi "k8s.io/api/core/v1"
+	informerfactory "k8s.io/client-go/informers"
+	v1coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+
+	netdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	netdefinformerv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1"
 )
 
+func newPodInformer(ctx context.Context, kclient kubernetes.Interface) cache.SharedIndexInformer {
+	informerFactory := informerfactory.NewSharedInformerFactory(kclient, 0*time.Second)
+
+	podInformer := informerFactory.InformerFor(&kapi.Pod{}, func(c kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return v1coreinformers.NewFilteredPodInformer(
+			c,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+			nil)
+	})
+
+	informerFactory.Start(ctx.Done())
+
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), podInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	return podInformer
+}
+
+func newNetDefInformer(ctx context.Context, client netdefclient.Interface) cache.SharedIndexInformer {
+	informerFactory := netdefinformer.NewSharedInformerFactory(client, 0*time.Second)
+
+	netdefInformer := informerFactory.InformerFor(&netdefv1.NetworkAttachmentDefinition{}, func(client netdefclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return netdefinformerv1.NewNetworkAttachmentDefinitionInformer(
+			client,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	})
+
+	informerFactory.Start(ctx.Done())
+
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), netdefInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	return netdefInformer
+}
+
 var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 	var testNS ns.NetNS
 	var tmpDir string
 	resultCNIVersion := "1.0.0"
 	configPath := "/tmp/foo.multus.conf"
+	var ctx context.Context
+	var cancel context.CancelFunc
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
@@ -52,9 +117,12 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		// Touch the default network file.
 		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 
+		ctx, cancel = context.WithCancel(context.TODO())
 	})
 
 	AfterEach(func() {
+		cancel()
+
 		// Cleanup default network file.
 		if _, errStat := os.Stat(configPath); errStat == nil {
 			errRemove := os.Remove(configPath)
@@ -76,7 +144,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -144,7 +212,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -192,6 +260,71 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(err).To(MatchError("[//:weave1]: error adding container to network \"weave1\": DelegateAdd: cannot set \"weave-net\" interface name to \"eth0\": validateIfName: no net namespace fsdadfad found: failed to Statfs \"fsdadfad\": no such file or directory"))
 	})
 
+	It("executes delegates (plugin without interface)", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.0.0",
+	        "type": "weave-net"
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.0.0",
+	        "type": "other-plugin"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.0.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// other1 just returns empty result
+		expectedResult2 := &cni100.Result{
+			CNIVersion: "0.4.0",
+		}
+		expectedConf2 := `{
+	    "name": "other1",
+	    "cniVersion": "1.0.0",
+	    "type": "other-plugin"
+	}`
+		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
+
+		result, err := CmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdCheck(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = CmdDel(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("returns the previous result using CmdCheck", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
@@ -200,7 +333,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -276,7 +409,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
@@ -345,7 +478,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 	    "name": "node-cni-network",
 	    "type": "multus",
-	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "readinessindicatorfile": "/tmp/foo.multus.conf",
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [%s,%s]
 	}`, expectedConf1, expectedConf2)),
@@ -391,7 +524,7 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 			StdinData: []byte(fmt.Sprintf(`{
 		    "name": "node-cni-network",
 		    "type": "multus",
-		    "defaultnetworkfile": "/tmp/foo.multus.conf",
+		    "readinessindicatorfile": "/tmp/foo.multus.conf",
 		    "defaultnetworkwaitseconds": 3,
 		    "delegates": [%s,%s]
 		}`, expectedConf1, expectedConf2)),
@@ -773,6 +906,116 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
+	It("executes clusterNetwork delegate with a shared informer", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		fKubeClient.AddPod(fakePod)
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		podInformer := newPodInformer(ctx, fKubeClient.Client)
+		netdefInformer := newNetDefInformer(ctx, fKubeClient.NetClient)
+		fKubeClient.SetK8sClientInformers(podInformer, netdefInformer)
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("executes clusterNetwork delegate with a shared informer if pod is not immediately found", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+		"name": "net1",
+		"type": "mynet",
+		"cniVersion": "1.0.0"
+	}`
+		expectedResult1 := &cni100.Result{
+			CNIVersion: "1.0.0",
+			IPs: []*cni100.IPConfig{{
+				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "defaultNetworks": [],
+	    "clusterNetwork": "net1",
+	    "delegates": []
+	}`),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := NewFakeClientInfo()
+		_, err := fKubeClient.AddNetAttachDef(testhelpers.NewFakeNetAttachDef("kube-system", "net1", net1))
+		Expect(err).NotTo(HaveOccurred())
+
+		podInformer := newPodInformer(ctx, fKubeClient.Client)
+		netdefInformer := newNetDefInformer(ctx, fKubeClient.NetClient)
+		fKubeClient.SetK8sClientInformers(podInformer, netdefInformer)
+
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			wg.Done()
+			time.Sleep(1 * time.Second)
+			fKubeClient.AddPod(fakePod)
+		}()
+		wg.Wait()
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(reflect.DeepEqual(result, expectedResult1)).To(BeTrue())
+
+		err = CmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
 	It("Verify the cache is created in dataDir", func() {
 		tmpCNIDir := tmpDir + "/cniData"
 		err := os.Mkdir(tmpCNIDir, 0777)
@@ -922,7 +1165,47 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
 	})
 
-	It("fails to execute confListDel given no 'plugins' key", func() {
+})
+
+var _ = Describe("multus operations cniVersion 1.1.0 config", func() {
+	var testNS ns.NetNS
+	var tmpDir string
+	configPath := "/tmp/foo.multus.conf"
+	var cancel context.CancelFunc
+
+	BeforeEach(func() {
+		// Create a new NetNS so we don't modify the host
+		var err error
+		testNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+		os.Setenv("CNI_NETNS", testNS.Path())
+		os.Setenv("CNI_PATH", "/some/path")
+
+		tmpDir, err = os.MkdirTemp("", "multus_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		// Touch the default network file.
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+		_, cancel = context.WithCancel(context.TODO())
+	})
+
+	AfterEach(func() {
+		cancel()
+
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+		Expect(testNS.Close()).To(Succeed())
+		os.Unsetenv("CNI_PATH")
+		os.Unsetenv("CNI_ARGS")
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes delegates with CNI Check", func() {
 		args := &skel.CmdArgs{
 			ContainerID: "123456789",
 			Netns:       testNS.Path(),
@@ -934,55 +1217,253 @@ var _ = Describe("multus operations cniVersion 1.0.0 config", func() {
 	    "defaultnetworkwaitseconds": 3,
 	    "delegates": [{
 	        "name": "weave1",
-	        "cniVersion": "1.0.0",
-	        "type": "weave-net"
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "weave-net"
+	        }]
 	    },{
 	        "name": "other1",
-	        "cniVersion": "1.0.0",
-	        "type": "other-plugin"
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "other-plugin"
+		}]
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedConf1 := `{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "", expectedConf1, nil, nil)
+
+		err := CmdStatus(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		// we only execute once for cluster network, not additional one
+		Expect(fExec.statusIndex).To(Equal(1))
+	})
+
+	It("returns empty add result using top-level cniVersion when pod is not found", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        "K8S_POD_NAME=missing-pod;K8S_POD_NAMESPACE=default",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+	    "cniVersion": "1.1.0",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
 	    }]
 	}`),
 		}
 
 		fExec := newFakeExec()
-		expectedResult1 := &cni100.Result{
-			CNIVersion: "1.0.0",
-			IPs: []*cni100.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.2/24"),
-			},
-			},
+		fKubeClient := NewFakeClientInfo()
+
+		result, err := CmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		r, ok := result.(*cni100.Result)
+		Expect(ok).To(BeTrue())
+		Expect(r.CNIVersion).To(Equal("1.1.0"))
+		Expect(fExec.addIndex).To(Equal(0))
+	})
+
+	It("propagates delegate STATUS errors", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
 		}
-		expectedConf1 := `{
+		k8sArgs := &types.K8sArgs{
+			K8S_POD_NAMESPACE:          cnitypes.UnmarshallableString("default"),
+			K8S_POD_NAME:               cnitypes.UnmarshallableString("pod"),
+			K8S_POD_INFRA_CONTAINER_ID: cnitypes.UnmarshallableString("sandbox"),
+			K8S_POD_UID:                cnitypes.UnmarshallableString("uid"),
+		}
+
+		delegateConf, err := types.LoadDelegateNetConf([]byte(`{
 	    "name": "weave1",
-	    "cniVersion": "1.0.0",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`), nil, "", "")
+		Expect(err).NotTo(HaveOccurred())
+		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, args.IfName, nil, delegateConf)
+
+		fExec := newFakeExec()
+		expectedConf := `{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
 	    "type": "weave-net"
 	}`
-		fExec.addPlugin100(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin100(nil, "", expectedConf, nil, &cnitypes.Error{Code: 50, Msg: "status failed"})
 
-		expectedResult2 := &cni100.Result{
-			CNIVersion: "1.0.0",
-			IPs: []*cni100.IPConfig{{
-				Address: *testhelpers.EnsureCIDR("1.1.1.5/24"),
-			},
-			},
-		}
-		expectedConf2 := `{
-	    "name": "other1",
-	    "cniVersion": "1.0.0",
-	    "type": "other-plugin"
-	}`
-		fExec.addPlugin100(nil, "net1", expectedConf2, expectedResult2, nil)
-
-		fakeMultusNetConf := types.NetConf{
-			BinDir: "/opt/cni/bin",
-		}
-		// use fExec for the exec param
-		rawnetconflist := []byte(`{"cniVersion":"1.0.0","name":"weave1","type":"weave-net"}`)
-		k8sargs, err := k8sclient.GetK8sArgs(args)
-		n, err := types.LoadNetConf(args.StdinData)
-		rt, _ := types.CreateCNIRuntimeConf(args, k8sargs, args.IfName, n.RuntimeConfig, nil)
-
-		err = conflistDel(rt, rawnetconflist, &fakeMultusNetConf, fExec)
+		err = DelegateStatus(fExec, delegateConf, rt, &types.NetConf{BinDir: "/bin", CNIDir: tmpDir})
 		Expect(err).To(HaveOccurred())
+		var cniErr *cnitypes.Error
+		Expect(errors.As(err, &cniErr)).To(BeTrue())
+		Expect(cniErr.Code).To(Equal(uint(50)))
+		Expect(cniErr.Msg).To(Equal("status failed"))
+	})
+
+	It("propagates CmdStatus errors for single plugin delegates", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
+	    }]
+	}`),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedConf := `{
+	    "name": "weave1",
+	    "cniVersion": "1.1.0",
+	    "type": "weave-net"
+	}`
+		fExec.addPlugin100(nil, "", expectedConf, nil, &cnitypes.Error{Code: 50, Msg: "status failed"})
+
+		err := CmdStatus(args, fExec, nil)
+		Expect(err).To(HaveOccurred())
+		var cniErr *cnitypes.Error
+		Expect(errors.As(err, &cniErr)).To(BeTrue())
+		Expect(cniErr.Code).To(Equal(uint(50)))
+		Expect(cniErr.Msg).To(Equal("status failed"))
+	})
+
+	It("executes delegates with CNI GC", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		cniCacheDir := filepath.Join(tmpCNIDir, "/results")
+		err = os.Mkdir(cniCacheDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		//create fake cniResult file
+		err = os.WriteFile(filepath.Join(cniCacheDir, "cbr0-3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755-eth0"), []byte(`{"kind":"cniCacheV1","containerId":"3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755","config":"eyJjbmlWZXJzaW9uIjoiMC4zLjEiLCJuYW1lIjoiY2JyMCIsInBsdWdpbnMiOlt7ImNhcGFiaWxpdGllcyI6eyJpby5rdWJlcm5ldGVzLmNyaS5wb2QtYW5ub3RhdGlvbnMiOnRydWV9LCJkZWxlZ2F0ZSI6eyJoYWlycGluTW9kZSI6dHJ1ZSwiaXNEZWZhdWx0R2F0ZXdheSI6dHJ1ZX0sInR5cGUiOiJmbGFubmVsIn0seyJjYXBhYmlsaXRpZXMiOnsicG9ydE1hcHBpbmdzIjp0cnVlfSwidHlwZSI6InBvcnRtYXAifV19","ifName":"eth0","networkName":"cbr0","netns":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e","cniArgs":[["IgnoreUnknown","true"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"],["IgnoreUnknown","1"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"]],"result":{"cniVersion":"0.3.1","dns":{},"interfaces":[{"mac":"ea:19:25:a2:a1:93","name":"cni0"},{"mac":"ba:76:61:2f:8b:ca","name":"vethc42d3d18"},{"mac":"7e:57:6a:9b:6b:b5","name":"eth0","sandbox":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e"}],"ips":[{"address":"10.244.1.4/24","gateway":"10.244.1.1","interface":2,"version":"4"}],"routes":[{"dst":"10.244.0.0/16"},{"dst":"0.0.0.0/0","gw":"10.244.1.1"}]}}`), 0666)
+		Expect(err).NotTo(HaveOccurred())
+		err = os.WriteFile(filepath.Join(cniCacheDir, "macvlan-conf-1-3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755-net1"), []byte(`{"kind":"cniCacheV1","containerId":"3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755","config":"eyJjbmlWZXJzaW9uIjoiMC4zLjEiLCJpcGFtIjp7ImFkZHJlc3NlcyI6W3siYWRkcmVzcyI6IjEwLjEuMS4xMDEvMjQifV0sInR5cGUiOiJzdGF0aWMifSwibWFzdGVyIjoiZXRoMSIsIm1vZGUiOiJicmlkZ2UiLCJuYW1lIjoibWFjdmxhbi1jb25mLTEiLCJ0eXBlIjoibWFjdmxhbiJ9","ifName":"net1","networkName":"macvlan-conf-1","netns":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e","cniArgs":[["IgnoreUnknown","true"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"],["IgnoreUnknown","1"],["K8S_POD_NAMESPACE","default"],["K8S_POD_NAME","macvlan"],["K8S_POD_INFRA_CONTAINER_ID","3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755"],["K8S_POD_UID","f0bfbd5b-096d-48ef-998c-da26743dd0cb"]],"result":{"cniVersion":"0.3.1","dns":{},"interfaces":[{"mac":"36:b3:c5:29:ad:b8","name":"net1","sandbox":"/var/run/netns/8b8677c8-8929-4746-8206-514069760f6e"}],"ips":[{"address":"10.1.1.101/24","interface":0,"version":"4"}]}}`), 0666)
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "weave-net"
+	        }]
+	    },{
+	        "name": "other1",
+	        "cniVersion": "1.1.0",
+		"plugins": [{
+	            "type": "other-plugin"
+		}]
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		logging.SetLogLevel("verbose")
+
+		fExec := newFakeExec()
+		expectedConf1 := `{
+			"cni.dev/attachments": [
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "eth0"
+				},
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "net1"
+				}
+			],
+			"cni.dev/valid-attachments": [
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "eth0"
+				},
+				{
+					"containerID": "3f6940ab5ab43bc522569d15b23f8c1bbde1d7678b080398506924fc01d72755",
+					"ifname": "net1"
+				}
+			],
+			"cniVersion": "1.1.0",
+			"name": "weave1",
+			"type": "weave-net"
+		}`
+		fExec.addPlugin100(nil, "", expectedConf1, nil, nil)
+
+		err = CmdGC(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		// we only execute once for cluster network, not additional one
+		Expect(fExec.gcIndex).To(Equal(1))
+		err = os.RemoveAll(tmpCNIDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes single plugin delegates with CNI GC", func() {
+		tmpCNIDir := tmpDir + "/cniData-single"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		cniCacheDir := filepath.Join(tmpCNIDir, "/results")
+		err = os.Mkdir(cniCacheDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+	    "name": "node-cni-network",
+	    "type": "multus",
+	    "defaultnetworkfile": "/tmp/foo.multus.conf",
+	    "defaultnetworkwaitseconds": 3,
+	    "cniDir": "%s",
+	    "delegates": [{
+	        "name": "weave1",
+	        "cniVersion": "1.1.0",
+	        "type": "weave-net"
+	    }]
+	}`, tmpCNIDir)),
+		}
+
+		fExec := newFakeExec()
+		fExec.addPlugin100(nil, "", "", nil, nil)
+
+		err = CmdGC(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.gcIndex).To(Equal(1))
+
+		err = os.RemoveAll(tmpCNIDir)
+		Expect(err).NotTo(HaveOccurred())
 	})
 })

pkg/multus/multus_suite_test.go

@@ -14,6 +14,8 @@
 
 package multus
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"bytes"
 	"context"
@@ -56,6 +58,8 @@ type fakeExec struct {
 	addIndex        int
 	delIndex        int
 	chkIndex        int
+	statusIndex     int
+	gcIndex         int
 	expectedDelSkip int
 	plugins         map[string]*fakePlugin
 }
@@ -166,6 +170,14 @@ func (f *fakeExec) ExecPlugin(_ context.Context, pluginPath string, stdinData []
 		Expect(len(f.plugins)).To(BeNumerically(">", f.delIndex))
 		index = len(f.plugins) - f.expectedDelSkip - f.delIndex - 1
 		f.delIndex++
+	case "GC":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.statusIndex))
+		index = f.gcIndex
+		f.gcIndex++
+	case "STATUS":
+		Expect(len(f.plugins)).To(BeNumerically(">", f.statusIndex))
+		index = f.statusIndex
+		f.statusIndex++
 	default:
 		// Should never be reached
 		Expect(false).To(BeTrue())
@@ -224,7 +236,7 @@ func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) {
 func NewFakeClientInfo() *k8sclient.ClientInfo {
 	return &k8sclient.ClientInfo{
 		Client:        fake.NewSimpleClientset(),
-		NetClient:     netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient:     netfake.NewSimpleClientset(),
 		EventRecorder: record.NewFakeRecorder(10),
 	}
 }

pkg/netutils/netutils.go

@@ -23,8 +23,10 @@ import (
 	"path/filepath"
 
 	"github.com/containernetworking/cni/libcni"
+	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 )
 
@@ -82,7 +84,12 @@ func SetDefaultGW(netnsPath string, ifName string, gateways []net.IP) error {
 			// Perform the creation of the default route....
 			err = netlink.RouteAdd(&newDefaultRoute)
 			if err != nil {
-				logging.Errorf("SetDefaultGW: Error adding route: %v", err)
+				if os.IsExist(err) || err == unix.EEXIST {
+					logging.Debugf("SetDefaultGW: Route already exists, ignoring: %v", err)
+					err = nil
+				} else {
+					logging.Errorf("SetDefaultGW: Error adding route: %v", err)
+				}
 			}
 		}
 		return err
@@ -139,6 +146,7 @@ func deleteDefaultGWCacheBytes(cacheFile []byte, ipv4, ipv6 bool) ([]byte, error
 }
 
 func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interface{}, error) {
+	var newRoutes []interface{}
 	for i, r := range routes {
 		route, ok := r.(map[string]interface{})
 		if !ok {
@@ -150,12 +158,12 @@ func deleteDefaultGWResultRoutes(routes []interface{}, dstGW string) ([]interfac
 			if !ok {
 				return nil, fmt.Errorf("wrong dst format: %v", route["dst"])
 			}
-			if dst == dstGW {
-				routes = append(routes[:i], routes[i+1:]...)
+			if dst != dstGW {
+				newRoutes = append(newRoutes, routes[i])
 			}
 		}
 	}
-	return routes, nil
+	return newRoutes, nil
 }
 
 func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[string]interface{}, error) {
@@ -176,7 +184,7 @@ func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[
 		return deleteDefaultGWResult020(result, ipv4, ipv6)
 	}
 
-	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+	if !isSupportedGatewayResultVersion(cniVersion) {
 		return nil, fmt.Errorf("not supported version: %s", cniVersion)
 	}
 
@@ -205,7 +213,12 @@ func deleteDefaultGWResult(result map[string]interface{}, ipv4, ipv6 bool) (map[
 			return nil, err
 		}
 	}
-	result["routes"] = routes
+
+	if len(routes) == 0 {
+		delete(result, "routes")
+	} else {
+		result["routes"] = routes
+	}
 
 	return result, nil
 }
@@ -328,7 +341,7 @@ func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]
 		return addDefaultGWResult020(result, gw)
 	}
 
-	if cniVersion != "0.3.0" && cniVersion != "0.3.1" && cniVersion != "0.4.0" && cniVersion != "1.0.0" {
+	if !isSupportedGatewayResultVersion(cniVersion) {
 		return nil, fmt.Errorf("not supported version: %s", cniVersion)
 	}
 
@@ -356,6 +369,19 @@ func addDefaultGWResult(result map[string]interface{}, gw []net.IP) (map[string]
 	return result, nil
 }
 
+func isSupportedGatewayResultVersion(cniVersion string) bool {
+	switch cniVersion {
+	case "0.3.0", "0.3.1", "0.4.0":
+		return true
+	}
+
+	if gt, _ := cniversion.GreaterThanOrEqualTo(cniVersion, "1.0.0"); gt {
+		return true
+	}
+
+	return false
+}
+
 func addDefaultGWResult020(result map[string]interface{}, gw []net.IP) (map[string]interface{}, error) {
 	for _, g := range gw {
 		if g.To4() != nil {

pkg/netutils/netutils_test.go

@@ -15,6 +15,8 @@
 
 package netutils
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"net"
@@ -639,6 +641,39 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(5))
 		})
 
+		It("verify ipv4 default gateway from single routes is removed/added from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10.1.1.103/24",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "0.0.0.0/0",
+        "gw": "10.1.1.1"
+      }
+    ]
+  }
+}`)
+			newResult1, err := deleteDefaultGWCacheBytes(origResult, true, false)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = addDefaultGWCacheBytes(newResult1, []net.IP{net.ParseIP("10.1.1.1")})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("verify ipv6 default gateway is removed from CNI 1.0.0 results", func() {
 			origResult := []byte(`{
   "kind": "cniCacheV1",
@@ -711,6 +746,39 @@ var _ = Describe("netutil cnicache function testing", func() {
 			Expect(len(result.Result.Routes)).To(Equal(5))
 		})
 
+		It("verify ipv6 default gateway from single routes is removed/added from CNI 1.0.0 results", func() {
+			origResult := []byte(`{
+  "kind": "cniCacheV1",
+  "result": {
+    "cniVersion": "1.0.0",
+    "dns": {},
+    "interfaces": [
+      {
+        "mac": "0a:c2:e6:3d:45:17",
+        "name": "net1",
+        "sandbox": "/run/netns/bb74fcb9-989a-4589-b2df-ddd0384a8ee5"
+      }
+    ],
+    "ips": [
+      {
+        "address": "10::1:1:103/64",
+        "interface": 0
+      }
+    ],
+    "routes": [
+      {
+        "dst": "::0/0",
+        "gw": "10::1:1:1"
+      }
+    ]
+  }
+}`)
+			newResult1, err := deleteDefaultGWCacheBytes(origResult, false, true)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = addDefaultGWCacheBytes(newResult1, []net.IP{net.ParseIP("10::1:1:1")})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("verify ipv4 default gateway is added to CNI 0.1.0/0.2.0 results without routes", func() {
 			origResult := []byte(`{
   "kind": "cniCacheV1",
@@ -1421,3 +1489,53 @@ var _ = Describe("netutil cnicache function testing", func() {
 
 	})
 })
+
+var _ = Describe("other function unit testing", func() {
+	It("deleteDefaultGWResultRoutes with invalid config", func() {
+		cniRouteConfig := []byte(`[
+		{ "dst": "0.0.0.0/0", "gw": "10.1.1.1" },
+		{ "dst": "10.1.1.0/24" },
+		{ "dst": "0.0.0.0/0", "gw": "10.1.1.1" }
+		]`)
+
+		var routes []interface{}
+		err := json.Unmarshal(cniRouteConfig, &routes)
+		Expect(err).NotTo(HaveOccurred())
+
+		newRoute, err := deleteDefaultGWResultRoutes(routes, "0.0.0.0/0")
+		Expect(err).NotTo(HaveOccurred())
+		routeJSON, err := json.Marshal(newRoute)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(routeJSON).Should(MatchJSON(`[{"dst":"10.1.1.0/24"}]`))
+	})
+
+	It("supports gateway result updates for cniVersion 1.1.0", func() {
+		deleteInput := map[string]interface{}{
+			"cniVersion": "1.1.0",
+			"routes": []interface{}{
+				map[string]interface{}{"dst": "0.0.0.0/0", "gw": "10.1.1.1"},
+			},
+		}
+		updatedDeleteResult, err := deleteDefaultGWResult(deleteInput, true, false)
+		Expect(err).NotTo(HaveOccurred())
+		_, hasRoutes := updatedDeleteResult["routes"]
+		Expect(hasRoutes).To(BeFalse())
+
+		addInput := map[string]interface{}{
+			"cniVersion": "1.1.0",
+		}
+		updatedAddResult, err := addDefaultGWResult(addInput, []net.IP{net.ParseIP("10.1.1.1")})
+		Expect(err).NotTo(HaveOccurred())
+		routes, ok := updatedAddResult["routes"].([]interface{})
+		Expect(ok).To(BeTrue())
+		Expect(routes).To(HaveLen(1))
+	})
+
+	It("rejects unsupported pre-1.0.0 cniVersion", func() {
+		addInput := map[string]interface{}{
+			"cniVersion": "0.9.0",
+		}
+		_, err := addDefaultGWResult(addInput, []net.IP{net.ParseIP("10.1.1.1")})
+		Expect(err).To(MatchError("not supported version: 0.9.0"))
+	})
+})

pkg/server/api/api.go

@@ -22,9 +22,19 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"time"
+
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
+	// APIReadyPollDuration specifies duration for API readiness check polling
+	APIReadyPollDuration = 100 * time.Millisecond
+	// APIReadyPollTimeout specifies timeout for API readiness check polling
+	APIReadyPollTimeout = 60000 * time.Millisecond
+
 	// MultusCNIAPIEndpoint is an endpoint for multus CNI request (for multus-shim)
 	MultusCNIAPIEndpoint = "/cni"
 	// MultusDelegateAPIEndpoint is an endpoint for multus delegate request (for hotplug)
@@ -45,7 +55,7 @@ func DoCNI(url string, req interface{}, socketPath string) ([]byte, error) {
 
 	client := &http.Client{
 		Transport: &http.Transport{
-			Dial: func(proto, addr string) (net.Conn, error) {
+			Dial: func(_, _ string) (net.Conn, error) {
 				return net.Dial("unix", socketPath)
 			},
 		},
@@ -63,6 +73,10 @@ func DoCNI(url string, req interface{}, socketPath string) ([]byte, error) {
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		cniErr := &cnitypes.Error{}
+		if err := json.Unmarshal(body, cniErr); err == nil && cniErr.Msg != "" {
+			return nil, cniErr
+		}
 		return nil, fmt.Errorf("CNI request failed with status %v: '%s'", resp.StatusCode, string(body))
 	}
 
@@ -88,3 +102,20 @@ func CreateDelegateRequest(cniCommand, cniContainerID, cniNetNS, cniIFName, podN
 		InterfaceAttributes: interfaceAttributes,
 	}
 }
+
+// WaitUntilAPIReady checks API readiness
+func WaitUntilAPIReady(socketPath string) error {
+	return utilwait.PollImmediate(APIReadyPollDuration, APIReadyPollTimeout, func() (bool, error) {
+		_, err := DoCNI(GetAPIEndpoint(MultusHealthAPIEndpoint), nil, SocketPath(socketPath))
+		return err == nil, nil
+	})
+}
+
+// CheckAPIReadyNow checks API readiness once
+func CheckAPIReadyNow(socketPath string) error {
+	_, err := DoCNI(GetAPIEndpoint(MultusHealthAPIEndpoint), nil, SocketPath(socketPath))
+	if err != nil {
+		return fmt.Errorf("CheckAPIReadyNow: Daemon not reachable over socketfile: %v", err)
+	}
+	return nil
+}

pkg/server/api/shim.go

@@ -16,6 +16,7 @@ package api
 
 import (
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"os"
 	"strings"
@@ -40,9 +41,12 @@ type ShimNetConf struct {
 	LogToStderr     bool   `json:"logToStderr,omitempty"`
 }
 
+// readyCheckFunc defines a type for API readiness check functions
+type readyCheckFunc func(string) error
+
 // CmdAdd implements the CNI spec ADD command handler
 func CmdAdd(args *skel.CmdArgs) error {
-	response, cniVersion, err := postRequest(args)
+	response, cniVersion, err := postRequest(args, WaitUntilAPIReady)
 	if err != nil {
 		return logging.Errorf("CmdAdd (shim): %v", err)
 	}
@@ -53,7 +57,7 @@ func CmdAdd(args *skel.CmdArgs) error {
 
 // CmdCheck implements the CNI spec CHECK command handler
 func CmdCheck(args *skel.CmdArgs) error {
-	_, _, err := postRequest(args)
+	_, _, err := postRequest(args, WaitUntilAPIReady)
 	if err != nil {
 		return logging.Errorf("CmdCheck (shim): %v", err)
 	}
@@ -63,27 +67,56 @@ func CmdCheck(args *skel.CmdArgs) error {
 
 // CmdDel implements the CNI spec DEL command handler
 func CmdDel(args *skel.CmdArgs) error {
-	_, _, err := postRequest(args)
+	_, _, err := postRequest(args, CheckAPIReadyNow)
 	if err != nil {
-		return logging.Errorf("CmdDel (shim): %v", err)
+		// No error in DEL (as of CNI spec)
+		logging.Errorf("CmdDel (shim): %v", err)
 	}
 	return nil
 }
 
-func postRequest(args *skel.CmdArgs) (*Response, string, error) {
+// CmdGC implements the CNI spec GC command handler
+func CmdGC(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args, WaitUntilAPIReady)
+	if err != nil {
+		return logging.Errorf("CmdGC (shim): %v", err)
+	}
+	return nil
+}
+
+// CmdStatus implements the CNI spec STATUS command handler
+func CmdStatus(args *skel.CmdArgs) error {
+	_, _, err := postRequest(args, WaitUntilAPIReady)
+	if err != nil {
+		return logging.Errorf("CmdStatus (shim): %v", err)
+	}
+	return nil
+}
+
+func postRequest(args *skel.CmdArgs, readinessCheck readyCheckFunc) (*Response, string, error) {
 	multusShimConfig, err := shimConfig(args.StdinData)
 	if err != nil {
 		return nil, "", fmt.Errorf("invalid CNI configuration passed to multus-shim: %w", err)
 	}
 
+	// Execute the readiness check as necessary (e.g. don't wait on CNI DEL)
+	if err := readinessCheck(multusShimConfig.MultusSocketDir); err != nil {
+		return nil, multusShimConfig.CNIVersion, err
+	}
+
 	cniRequest, err := newCNIRequest(args)
 	if err != nil {
 		return nil, multusShimConfig.CNIVersion, err
 	}
 
-	body, err := DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
+	var body []byte
+	body, err = DoCNI("http://dummy/cni", cniRequest, SocketPath(multusShimConfig.MultusSocketDir))
 	if err != nil {
-		return nil, multusShimConfig.CNIVersion, err
+		var cniErr *cnitypes.Error
+		if stderrors.As(err, &cniErr) {
+			return nil, multusShimConfig.CNIVersion, err
+		}
+		return nil, multusShimConfig.CNIVersion, fmt.Errorf("%s: StdinData: %s", err.Error(), string(args.StdinData))
 	}
 
 	response := &Response{}

pkg/server/config/config_suite_test.go

@@ -14,6 +14,8 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/server/config/generator.go

@@ -55,7 +55,8 @@ type MultusConf struct {
 	Type                     string              `json:"type"`
 	CniDir                   string              `json:"cniDir,omitempty"`
 	CniConfigDir             string              `json:"cniConfigDir,omitempty"`
-	SocketDir                string              `json:"socketDir,omitempty"`
+	AuxiliaryCNIChainName    string              `json:"auxiliaryCNIChainName,omitempty"`
+	DaemonSocketDir          string              `json:"daemonSocketDir,omitempty"`
 	MultusConfigFile         string              `json:"multusConfigFile,omitempty"`
 	MultusMasterCni          string              `json:"multusMasterCNI,omitempty"`
 	MultusAutoconfigDir      string              `json:"multusAutoconfigDir,omitempty"`
@@ -126,6 +127,11 @@ func (mc *MultusConf) Generate() (string, error) {
 	mc.CniConfigDir = ""
 	mc.MultusConfigFile = ""
 	mc.MultusAutoconfigDir = ""
+	mc.MultusMasterCni = ""
+	mc.ForceCNIVersion = false
+	// Readiness indicator file existence is already handled by the
+	// ConfigManager via an fsnotify watch, so CmdAdd/CmdDel don't need to.
+	mc.ReadinessIndicatorFile = ""
 
 	data, err := json.Marshal(mc)
 	return string(data), err

pkg/server/config/generator_test.go

@@ -14,6 +14,8 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"fmt"

pkg/server/config/manager.go

@@ -15,9 +15,12 @@
 package config
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"sync"
 
 	"github.com/fsnotify/fsnotify"
 
@@ -34,39 +37,43 @@ const (
 // Manager monitors the configuration of the primary CNI plugin, and
 // regenerates multus configuration whenever it gets updated.
 type Manager struct {
-	cniConfigData        map[string]interface{}
-	configWatcher        *fsnotify.Watcher
-	multusConfig         *MultusConf
-	multusConfigDir      string
-	multusConfigFilePath string
-	primaryCNIConfigPath string
+	cniConfigData              map[string]interface{}
+	configWatcher              *fsnotify.Watcher
+	multusConfig               *MultusConf
+	multusConfigDir            string
+	multusConfigFilePath       string
+	readinessIndicatorFilePath string
+	primaryCNIConfigPath       string
 }
 
 // NewManager returns a config manager object, configured to read the
-// primary CNI configuration in `multusAutoconfigDir`. This constructor will auto-discover
-// the primary CNI for which it will delegate.
-func NewManager(config MultusConf, multusAutoconfigDir string, forceCNIVersion bool) (*Manager, error) {
-	defaultCNIPluginName, err := getPrimaryCNIPluginName(multusAutoconfigDir)
-	if err != nil {
-		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
-		return nil, err
+// primary CNI configuration in `config.MultusAutoconfigDir`. If
+// `config.MultusMasterCni` is empty, this constructor will auto-discover the
+// primary CNI for which it will delegate.
+func NewManager(config MultusConf) (*Manager, error) {
+	var err error
+	defaultPluginName := config.MultusMasterCni
+	if defaultPluginName == "" {
+		defaultPluginName, err = getPrimaryCNIPluginName(config.MultusAutoconfigDir)
+		if err != nil {
+			_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+			return nil, err
+		}
 	}
-	return newManager(config, multusAutoconfigDir, defaultCNIPluginName, forceCNIVersion)
-}
 
-// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
-// configured to persist the configuration to `multusAutoconfigDir`. This
-// constructor will use the primary CNI plugin indicated by the user, via the
-// primaryCNIPluginName variable.
-func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir, primaryCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
-	return newManager(config, multusAutoconfigDir, primaryCNIPluginName, forceCNIVersion)
+	return newManager(config, defaultPluginName)
 }
 
 // overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case
 func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
-	masterCNIConfigData, err := os.ReadFile(cniConfigFile)
+	path, err := filepath.Abs(cniConfigFile)
 	if err != nil {
-		return fmt.Errorf("failed to read cni config %s: %v", cniConfigFile, err)
+		return fmt.Errorf("illegal path %s in cni config path %s: %w", path, cniConfigFile, err)
+	}
+
+	masterCNIConfigData, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read cni config %s: %v", path, err)
 	}
 
 	var primaryCNIConfigData map[string]interface{}
@@ -80,45 +87,85 @@ func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
 		return fmt.Errorf("couldn't update cluster network config: %v", err)
 	}
 
-	err = os.WriteFile(cniConfigFile, configBytes, 0644)
+	err = os.WriteFile(path, configBytes, 0644)
 	if err != nil {
 		return fmt.Errorf("couldn't update cluster network config: %v", err)
 	}
 	return nil
 }
 
-func newManager(config MultusConf, multusConfigDir, defaultCNIPluginName string, forceCNIVersion bool) (*Manager, error) {
-	if forceCNIVersion {
-		err := overrideCNIVersion(cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName), config.CNIVersion)
+func newManager(config MultusConf, defaultCNIPluginName string) (*Manager, error) {
+	if config.ForceCNIVersion {
+		err := overrideCNIVersion(filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName), config.CNIVersion)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	watcher, err := newWatcher(multusConfigDir)
+	readinessIndicatorPath := ""
+	if config.ReadinessIndicatorFile != "" {
+		readinessIndicatorPath = filepath.Dir(config.ReadinessIndicatorFile)
+	}
+
+	watcher, err := newWatcher(config.MultusAutoconfigDir, readinessIndicatorPath)
 	if err != nil {
 		return nil, err
 	}
 
-	if defaultCNIPluginName == fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName) {
-		return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", multusConfigDir, multusConfigFileName)
+	if defaultCNIPluginName == fmt.Sprintf("%s/%s", config.MultusAutoconfigDir, multusConfigFileName) {
+		return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", config.MultusAutoconfigDir, multusConfigFileName)
 	}
 
 	configManager := &Manager{
-		configWatcher:        watcher,
-		multusConfig:         &config,
-		multusConfigDir:      multusConfigDir,
-		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
-		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+		configWatcher:              watcher,
+		multusConfig:               &config,
+		multusConfigDir:            config.MultusAutoconfigDir,
+		multusConfigFilePath:       filepath.Join(config.CniConfigDir, multusConfigFileName),
+		primaryCNIConfigPath:       filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName),
+		readinessIndicatorFilePath: config.ReadinessIndicatorFile,
 	}
 
 	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
 		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
 	}
 
+	if config.OverrideNetworkName {
+		if err := configManager.overrideNetworkName(); err != nil {
+			return nil, logging.Errorf("could not override the network name: %v", err)
+		}
+	}
+
 	return configManager, nil
 }
 
+// Start generates an updated Multus config, writes it, and begins watching
+// the config directory and readiness indicator files for changes
+func (m *Manager) Start(ctx context.Context, wg *sync.WaitGroup) error {
+	generatedMultusConfig, err := m.GenerateConfig()
+	if err != nil {
+		return logging.Errorf("failed to generated the multus configuration: %v", err)
+	}
+	logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+	multusConfigFile, err := m.PersistMultusConfig(generatedMultusConfig)
+	if err != nil {
+		return logging.Errorf("failed to persist the multus configuration: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := m.monitorPluginConfiguration(ctx); err != nil {
+			_ = logging.Errorf("error watching file: %v", err)
+		}
+		logging.Verbosef("ConfigWatcher done")
+		logging.Verbosef("Delete old config @ %v", multusConfigFile)
+		os.Remove(multusConfigFile)
+	}()
+
+	return nil
+}
+
 func (m *Manager) loadPrimaryCNIConfigFromFile() error {
 	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
 	if err != nil {
@@ -132,9 +179,9 @@ func (m *Manager) loadPrimaryCNIConfigFromFile() error {
 	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
 }
 
-// OverrideNetworkName overrides the name of the multus configuration with the
+// overrideNetworkName overrides the name of the multus configuration with the
 // name of the delegated primary CNI.
-func (m *Manager) OverrideNetworkName() error {
+func (m *Manager) overrideNetworkName() error {
 	name, ok := m.cniConfigData["name"]
 	if !ok {
 		return fmt.Errorf("failed to access delegate CNI plugin name")
@@ -157,7 +204,7 @@ func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface
 }
 
 // GenerateConfig generates a multus configuration from its current state
-func (m Manager) GenerateConfig() (string, error) {
+func (m *Manager) GenerateConfig() (string, error) {
 	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
 		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
 		return "", nil
@@ -165,25 +212,25 @@ func (m Manager) GenerateConfig() (string, error) {
 	return m.multusConfig.Generate()
 }
 
-// MonitorPluginConfiguration monitors the configuration file pointed
+// monitorPluginConfiguration monitors the configuration file pointed
 // to by the primaryCNIPluginName attribute, and re-generates the multus
 // configuration whenever the primary CNI config is updated.
-func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<- struct{}) error {
+func (m *Manager) monitorPluginConfiguration(ctx context.Context) error {
 	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
 
 	for {
 		select {
 		case event := <-m.configWatcher.Events:
-			// we're watching the DIR where the config sits, and the event
-			// does not concern the primary CNI config. Skip it.
-			if event.Name != m.primaryCNIConfigPath {
-				logging.Debugf("skipping un-related event %v", event)
+			if !m.shouldRegenerateConfig(event) {
 				continue
 			}
 			logging.Debugf("process event: %v", event)
 
-			if !shouldRegenerateConfig(event) {
-				continue
+			// if readinessIndicatorFile is removed, then restart multus
+			if m.readinessIndicatorFilePath != "" && m.readinessIndicatorFilePath == event.Name {
+				logging.Verbosef("readiness indicator file is gone. restart multus-daemon")
+				os.Remove(m.multusConfigFilePath)
+				os.Exit(2)
 			}
 
 			updatedConfig, err := m.GenerateConfig()
@@ -192,7 +239,7 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 			}
 
 			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
-			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+			if _, err := m.PersistMultusConfig(updatedConfig); err != nil {
 				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
 			}
 			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
@@ -205,10 +252,9 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 			}
 			logging.Errorf("CNI monitoring error %v", err)
 
-		case <-shutDown:
+		case <-ctx.Done():
 			logging.Verbosef("Stopped monitoring, closing channel ...")
 			_ = m.configWatcher.Close()
-			close(done)
 			return nil
 		}
 	}
@@ -216,9 +262,28 @@ func (m Manager) MonitorPluginConfiguration(shutDown <-chan struct{}, done chan<
 
 // PersistMultusConfig persists the provided configuration to the disc, with
 // Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
-func (m Manager) PersistMultusConfig(config string) error {
-	logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
-	return os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
+func (m *Manager) PersistMultusConfig(config string) (string, error) {
+	if _, err := os.Stat(m.multusConfigFilePath); err == nil {
+		logging.Debugf("Overwriting Multus CNI configuration @ %s", m.multusConfigFilePath)
+	} else {
+		logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
+	}
+	return m.multusConfigFilePath, os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
+}
+
+func (m *Manager) shouldRegenerateConfig(event fsnotify.Event) bool {
+	// first, check the readiness indicator file existence
+	if event.Name == m.readinessIndicatorFilePath {
+		return event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename)
+	}
+
+	// we're watching the DIR where the config sits, and the event
+	// does not concern the primary CNI config. Skip it.
+	if event.Name == m.primaryCNIConfigPath {
+		return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
+	}
+	logging.Debugf("skipping un-related event %v", event)
+	return false
 }
 
 func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
@@ -229,11 +294,7 @@ func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
 	return masterCniConfigFileName, nil
 }
 
-func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
-	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
-}
-
-func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+func newWatcher(cniConfigDir string, readinessIndicatorDir string) (*fsnotify.Watcher, error) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
@@ -246,16 +307,18 @@ func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
 	}()
 
 	if err = watcher.Add(cniConfigDir); err != nil {
-		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+		return nil, fmt.Errorf("failed to add watch on %q for cni config: %v", cniConfigDir, err)
+	}
+	// if readinessIndicatorDir is different from cniConfigDir,
+	if readinessIndicatorDir != "" && cniConfigDir != readinessIndicatorDir {
+		if err = watcher.Add(readinessIndicatorDir); err != nil {
+			return nil, fmt.Errorf("failed to add watch on %q for readinessIndicator: %v", readinessIndicatorDir, err)
+		}
 	}
 
 	return watcher, nil
 }
 
-func shouldRegenerateConfig(event fsnotify.Event) bool {
-	return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
-}
-
 func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
 	masterCNIConfigData, err := os.ReadFile(masterCNIPluginPath)
 	if err != nil {

pkg/server/config/manager_test.go

@@ -14,11 +14,14 @@
 
 package config
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
-	"time"
+	"sync"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -41,6 +44,7 @@ var _ = Describe("Configuration Manager", func() {
 	var configManager *Manager
 	var multusConfigDir string
 	var defaultCniConfig string
+	var wg *sync.WaitGroup
 
 	BeforeEach(func() {
 		var err error
@@ -53,19 +57,25 @@ var _ = Describe("Configuration Manager", func() {
 
 		multusConfFile := fmt.Sprintf(`{
 			"name": %q,
-			"cniVersion": %q
-		}`, defaultCniConfig, cniVersion)
+			"cniVersion": %q,
+			"multusAutoconfigDir": %q,
+			"multusMasterCNI": %q,
+			"forceCNIVersion": false
+		}`, defaultCniConfig, cniVersion, multusConfigDir, primaryCNIPluginName)
 		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
 		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
 
 		multusConf, err := ParseMultusConfig(multusConfFileName)
 		Expect(err).NotTo(HaveOccurred())
 
-		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		configManager, err = NewManager(*multusConf)
 		Expect(err).NotTo(HaveOccurred())
+
+		wg = &sync.WaitGroup{}
 	})
 
 	AfterEach(func() {
+		wg.Wait()
 		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
 	})
 
@@ -95,23 +105,17 @@ var _ = Describe("Configuration Manager", func() {
 	})
 
 	It("Check MonitorPluginConfiguration", func() {
-		config, err := configManager.GenerateConfig()
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		err := configManager.Start(ctx, wg)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = configManager.PersistMultusConfig(config)
-		Expect(err).NotTo(HaveOccurred())
-
-		configWatcherDoneChannel := make(chan struct{})
-		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
-			err := configManager.MonitorPluginConfiguration(configWatcherDoneChannel, stopChannel)
-			Expect(err).NotTo(HaveOccurred())
-		}(make(chan struct{}), configWatcherDoneChannel)
-
 		updatedCNIConfig := `
 {
   "cniVersion": "0.4.0",
   "name": "mycni-name",
   "type": "mycni2",
+  "capabilities": {"portMappings": true},
   "ipam": {},
   "dns": {}
 }
@@ -120,18 +124,16 @@ var _ = Describe("Configuration Manager", func() {
 		Expect(os.WriteFile(defaultCniConfig, []byte(updatedCNIConfig), UserRWPermission)).To(Succeed())
 
 		// wait for a while to get fsnotify event
-		time.Sleep(100 * time.Millisecond)
-		file, err := os.ReadFile(configManager.multusConfigFilePath)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(string(file)).To(Equal(config))
-
-		// stop groutine
-		configWatcherDoneChannel <- struct{}{}
+		Eventually(func() string {
+			file, err := os.ReadFile(configManager.multusConfigFilePath)
+			Expect(err).NotTo(HaveOccurred())
+			return string(file)
+		}, 2).Should(ContainSubstring("portMappings"))
 	})
 
 	When("the user requests the name of the multus configuration to be overridden", func() {
 		BeforeEach(func() {
-			Expect(configManager.OverrideNetworkName()).To(Succeed())
+			Expect(configManager.overrideNetworkName()).To(Succeed())
 		})
 
 		It("Overrides the name of the multus configuration when requested", func() {
@@ -171,14 +173,17 @@ var _ = Describe("Configuration Manager with mismatched cniVersion", func() {
 
 		multusConfFile := fmt.Sprintf(`{
 			"name": %q,
-			"cniVersion": %q
-		}`, defaultCniConfig, cniVersion)
+			"cniVersion": %q,
+			"multusAutoconfigDir": %q,
+			"multusMasterCNI": %q,
+			"forceCNIVersion": false
+		}`, defaultCniConfig, cniVersion, multusConfigDir, primaryCNIPluginName)
 		multusConfFileName := fmt.Sprintf("%s/10-testcni.conf", multusConfigDir)
 		Expect(os.WriteFile(multusConfFileName, []byte(multusConfFile), 0755)).To(Succeed())
 
 		multusConf, err := ParseMultusConfig(multusConfFileName)
 		Expect(err).NotTo(HaveOccurred())
-		_, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName, false)
+		_, err = NewManager(*multusConf)
 		Expect(err).To(MatchError("failed to load the primary CNI configuration as a multus delegate with error 'delegate cni version is 0.3.1 while top level cni version is 0.4.0'"))
 	})
 

pkg/server/exec_chroot.go

@@ -20,10 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"os/exec"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -34,91 +32,24 @@ import (
 
 // ChrootExec implements invoke.Exec to execute CNI with chroot
 type ChrootExec struct {
-	Stderr     io.Writer
-	chrootDir  string
-	workingDir string   // working directory in the outer root
-	outerRoot  *os.File // outer root directory
+	Stderr    io.Writer
+	chrootDir string
 	version.PluginDecoder
-	mu sync.Mutex
 }
 
 var _ invoke.Exec = &ChrootExec{}
 
-func (e *ChrootExec) chroot() error {
-	var err error
-	e.workingDir, err = os.Getwd()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
-		return fmt.Errorf("getwd before chroot failed: %v", err)
-	}
-
-	e.outerRoot, err = os.Open("/")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "getwd before chroot failed: %v\n", err)
-		return fmt.Errorf("getwd before chroot failed: %v", err)
-	}
-
-	if err := syscall.Chroot(e.chrootDir); err != nil {
-		fmt.Fprintf(os.Stderr, "chroot to %s failed: %v\n", e.chrootDir, err)
-		return fmt.Errorf("chroot to %s failed: %v", e.chrootDir, err)
-	}
-
-	if err := os.Chdir("/"); err != nil {
-		fmt.Fprintf(os.Stderr, "chdir to \"/\" failed: %v\n", err)
-		return fmt.Errorf("chdir to \"/\" failed: %v", err)
-	}
-
-	return nil
-}
-
-func (e *ChrootExec) escape() error {
-	if e.outerRoot == nil || e.workingDir == "" {
-		return nil
-	}
-
-	// change directory to outer root and close it
-	if err := syscall.Fchdir(int(e.outerRoot.Fd())); err != nil {
-		fmt.Fprintf(os.Stderr, "changing directory to outer root failed: %v\n", err)
-		return fmt.Errorf("changing directory to outer root failed: %v", err)
-	}
-
-	if err := e.outerRoot.Close(); err != nil {
-		fmt.Fprintf(os.Stderr, "closing outer root failed: %v\n", err)
-		return fmt.Errorf("closing outer root failed: %v", err)
-	}
-
-	// chroot to current directory aka "." being the outer root
-	if err := syscall.Chroot("."); err != nil {
-		fmt.Fprintf(os.Stderr, "chroot to current directory failed: %v\n", err)
-		return fmt.Errorf("chroot to current directory failed: %v", err)
-	}
-
-	if err := os.Chdir(e.workingDir); err != nil {
-		fmt.Fprintf(os.Stderr, "chdir to working directory failed: %v\n", err)
-		return fmt.Errorf("chdir to working directory failed: %v", err)
-	}
-	e.outerRoot = nil
-	e.workingDir = ""
-
-	return nil
-}
-
 // ExecPlugin executes CNI plugin with given environment/stdin data.
 func (e *ChrootExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
-	// lock and do chroot to execute plugin with host root
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	err := e.chroot()
-	defer e.escape()
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "ExecPlugin failed at chroot: %v\n", err)
-		return nil, fmt.Errorf("ExecPlugin failed at chroot: %v", err)
-	}
+	var err error
 
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}
 	c := exec.CommandContext(ctx, pluginPath)
+	// execute delegate CNI with host filesystem context.
+	c.SysProcAttr = &syscall.SysProcAttr{
+		Chroot: e.chrootDir,
+	}
 	c.Env = environ
 	c.Stdin = bytes.NewBuffer(stdinData)
 	c.Stdout = stdout
@@ -169,14 +100,5 @@ func (e *ChrootExec) pluginErr(err error, stdout, stderr []byte) error {
 
 // FindInPath try to find CNI plugin based on given path
 func (e *ChrootExec) FindInPath(plugin string, paths []string) (string, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	err := e.chroot()
-	defer e.escape()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "FindInPath failed at chroot: %v\n", err)
-		return "", fmt.Errorf("FindInPath failed at chroot: %v", err)
-	}
-
 	return invoke.FindInPath(plugin, paths)
 }

pkg/server/exec_chroot_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"os"
@@ -43,15 +45,4 @@ var _ = Describe("exec_chroot", func() {
 		_, err := chrootExec.ExecPlugin(context.Background(), "/bin/true", nil, nil)
 		Expect(err).To(HaveOccurred())
 	})
-
-	It("Call ChrootExec.FindInPath with dummy", func() {
-		chrootExec := &ChrootExec{
-			Stderr:    os.Stderr,
-			chrootDir: "/usr/bin",
-		}
-
-		_, err := chrootExec.FindInPath("true", []string{"/"})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
 })

pkg/server/server.go

@@ -15,14 +15,16 @@
 package server
 
 import (
-	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -38,6 +40,24 @@ import (
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+
+	netdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	netdefinformerv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1"
+
+	kapi "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+	informerfactory "k8s.io/client-go/informers"
+	v1coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/informers/internalinterfaces"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
 )
 
 const (
@@ -57,61 +77,62 @@ func FilesystemPreRequirements(rundir string) error {
 	return nil
 }
 
+func printCmdArgs(args *skel.CmdArgs) string {
+	return fmt.Sprintf("ContainerID:%q Netns:%q IfName:%q Args:%q Path:%q",
+		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
+}
+
 // HandleCNIRequest is the CNI server handler function; it is invoked whenever
 // a CNI request is processed.
-func (s *Server) HandleCNIRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+func (s *Server) HandleCNIRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs) ([]byte, error) {
 	var result []byte
 	var err error
 
-	logging.Verbosef("%s starting CNI request %+v", cmd, cniCmdArgs)
+	logging.Verbosef("%s starting CNI request %s", cmd, printCmdArgs(cniCmdArgs))
 	switch cmd {
 	case "ADD":
-		result, err = cmdAdd(cniCmdArgs, k8sArgs, exec, kubeClient)
+		result, err = s.cmdAdd(cniCmdArgs, k8sArgs)
 	case "DEL":
-		err = cmdDel(cniCmdArgs, k8sArgs, exec, kubeClient)
+		err = s.cmdDel(cniCmdArgs, k8sArgs)
 	case "CHECK":
-		err = cmdCheck(cniCmdArgs, k8sArgs, exec, kubeClient)
+		err = s.cmdCheck(cniCmdArgs, k8sArgs)
+	case "GC":
+		err = s.cmdGC(cniCmdArgs, k8sArgs)
+	case "STATUS":
+		err = s.cmdStatus(cniCmdArgs, k8sArgs)
 	default:
 		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
 	}
-	logging.Verbosef("%s finished CNI request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
-	if err != nil {
-		// Prefix errors with request info for easier failure debugging
-		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
-	}
-	return result, nil
+	logging.Verbosef("%s finished CNI request %s, result: %q, err: %v", cmd, printCmdArgs(cniCmdArgs), string(result), err)
+	return result, err
 }
 
 // HandleDelegateRequest is the CNI server handler function; it is invoked whenever
 // a CNI request is processed as delegate CNI request.
-func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+func (s *Server) HandleDelegateRequest(cmd string, k8sArgs *types.K8sArgs, cniCmdArgs *skel.CmdArgs, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
 	var result []byte
 	var err error
-	var multusConfByte []byte
 
-	multusConfByte = bytes.Replace(s.serverConfig, []byte(","), []byte("{"), 1)
 	multusConfig := types.GetDefaultNetConf()
-	if err = json.Unmarshal(multusConfByte, multusConfig); err != nil {
+	if err = json.Unmarshal(s.serverConfig, multusConfig); err != nil {
 		return nil, err
 	}
 
-	logging.Verbosef("%s starting delegate request %+v", cmd, cniCmdArgs)
+	logging.Verbosef("%s starting delegate request %s", cmd, printCmdArgs(cniCmdArgs))
 	switch cmd {
 	case "ADD":
-		result, err = cmdDelegateAdd(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig, interfaceAttributes)
+		result, err = s.cmdDelegateAdd(cniCmdArgs, k8sArgs, multusConfig, interfaceAttributes)
 	case "DEL":
-		err = cmdDelegateDel(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+		err = s.cmdDelegateDel(cniCmdArgs, k8sArgs, multusConfig)
 	case "CHECK":
-		err = cmdDelegateCheck(cniCmdArgs, k8sArgs, exec, kubeClient, multusConfig)
+		err = s.cmdDelegateCheck(cniCmdArgs, k8sArgs, multusConfig)
+	case "STATUS":
+		err = s.cmdDelegateStatus(cniCmdArgs, k8sArgs, multusConfig)
 	default:
 		return []byte(""), fmt.Errorf("unknown cmd type: %s", cmd)
 	}
-	logging.Verbosef("%s finished Delegate request %+v, result: %q, err: %v", cmd, *cniCmdArgs, string(result), err)
-	if err != nil {
-		// Prefix errors with request info for easier failure debugging
-		return nil, fmt.Errorf("%+v ERRORED: %v", *cniCmdArgs, err)
-	}
-	return result, nil
+	logging.Verbosef("%s finished Delegate request %s, result: %q, err: %v", cmd, printCmdArgs(cniCmdArgs), string(result), err)
+	return result, err
 }
 
 // GetListener creates a listener to a unix socket located in `socketPath`
@@ -127,11 +148,104 @@ func GetListener(socketPath string) (net.Listener, error) {
 	return l, nil
 }
 
+// Informer transform to trim object fields for memory efficiency.
+func informerObjectTrim(obj interface{}) (interface{}, error) {
+	if accessor, err := meta.Accessor(obj); err == nil {
+		accessor.SetManagedFields(nil)
+	}
+	if pod, ok := obj.(*kapi.Pod); ok {
+		pod.Spec.Volumes = []kapi.Volume{}
+		for i := range pod.Spec.Containers {
+			pod.Spec.Containers[i].Command = nil
+			pod.Spec.Containers[i].Args = nil
+			pod.Spec.Containers[i].Env = nil
+			pod.Spec.Containers[i].VolumeMounts = nil
+		}
+	}
+	return obj, nil
+}
+
+func newNetDefInformer(netdefClient netdefclient.Interface) (netdefinformer.SharedInformerFactory, cache.SharedIndexInformer) {
+	const resyncInterval time.Duration = 1 * time.Second
+
+	informerFactory := netdefinformer.NewSharedInformerFactoryWithOptions(netdefClient, resyncInterval)
+	netdefInformer := informerFactory.InformerFor(&netdefv1.NetworkAttachmentDefinition{}, func(client netdefclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return netdefinformerv1.NewNetworkAttachmentDefinitionInformer(
+			client,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	})
+
+	return informerFactory, netdefInformer
+}
+
+func newPodInformer(kubeClient kubernetes.Interface, nodeName string) (internalinterfaces.SharedInformerFactory, cache.SharedIndexInformer) {
+	var tweakFunc internalinterfaces.TweakListOptionsFunc
+	if nodeName != "" {
+		logging.Verbosef("Filtering pod watch for node %q", nodeName)
+		// Only watch for local pods
+		tweakFunc = func(opts *metav1.ListOptions) {
+			opts.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", nodeName).String()
+		}
+	}
+
+	const resyncInterval time.Duration = 1 * time.Second
+
+	informerFactory := informerfactory.NewSharedInformerFactoryWithOptions(kubeClient, resyncInterval, informerfactory.WithTransform(informerObjectTrim))
+	podInformer := informerFactory.InformerFor(&kapi.Pod{}, func(c kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+		return v1coreinformers.NewFilteredPodInformer(
+			c,
+			kapi.NamespaceAll,
+			resyncPeriod,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+			tweakFunc)
+	})
+
+	return informerFactory, podInformer
+}
+
+func isPerNodeCertEnabled(config *PerNodeCertificate) (bool, error) {
+	if config != nil && config.Enabled {
+		if config.BootstrapKubeconfig != "" && config.CertDir != "" {
+			return true, nil
+		}
+		return true, logging.Errorf("failed to configure PerNodeCertificate: enabled: %v, BootstrapKubeconfig: %q, CertDir: %q", config.Enabled, config.BootstrapKubeconfig, config.CertDir)
+	}
+	return false, nil
+}
+
 // NewCNIServer creates and returns a new Server object which will listen on a socket in the given path
-func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte) (*Server, error) {
-	kubeClient, err := k8s.InClusterK8sClient()
-	if err != nil {
-		return nil, fmt.Errorf("error getting k8s client: %v", err)
+func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte, ignoreReadinessIndicator bool) (*Server, error) {
+	var kubeClient *k8s.ClientInfo
+	enabled, err := isPerNodeCertEnabled(daemonConfig.PerNodeCertificate)
+	if enabled {
+		if err != nil {
+			return nil, err
+		}
+		perNodeCertConfig := daemonConfig.PerNodeCertificate
+		nodeName := os.Getenv("MULTUS_NODE_NAME")
+		if nodeName == "" {
+			return nil, logging.Errorf("error getting node name for perNodeCertificate, please check manifest to have MULTUS_NODE_NAME")
+		}
+
+		certDuration := DefaultCertDuration
+		if perNodeCertConfig.CertDuration != "" {
+			certDuration, err = time.ParseDuration(perNodeCertConfig.CertDuration)
+			if err != nil {
+				return nil, logging.Errorf("failed to parse certDuration: %v", err)
+			}
+		}
+
+		kubeClient, err = k8s.PerNodeK8sClient(nodeName, perNodeCertConfig.BootstrapKubeconfig, certDuration, perNodeCertConfig.CertDir)
+		if err != nil {
+			return nil, logging.Errorf("error getting perNodeClient: %v", err)
+		}
+	} else {
+		kubeClient, err = k8s.InClusterK8sClient()
+		if err != nil {
+			return nil, fmt.Errorf("error getting k8s client: %v", err)
+		}
 	}
 
 	exec := invoke.Exec(nil)
@@ -140,21 +254,17 @@ func NewCNIServer(daemonConfig *ControllerNetConf, serverConfig []byte) (*Server
 			Stderr:    os.Stderr,
 			chrootDir: daemonConfig.ChrootDir,
 		}
-		types.ChrootMutex = &chrootExec.mu
 		exec = chrootExec
 		logging.Verbosef("server configured with chroot: %s", daemonConfig.ChrootDir)
 	}
 
-	return newCNIServer(daemonConfig.SocketDir, kubeClient, exec, serverConfig)
+	return newCNIServer(daemonConfig.SocketDir, kubeClient, exec, serverConfig, ignoreReadinessIndicator)
 }
 
-func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte) (*Server, error) {
-
-	// preprocess server config to be used to override multus CNI config
-	// see extractCniData() for the detail
-	if servConfig != nil {
-		servConfig = bytes.Replace(servConfig, []byte("{"), []byte(","), 1)
-	}
+func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, servConfig []byte, ignoreReadinessIndicator bool) (*Server, error) {
+	informerFactory, podInformer := newPodInformer(kubeClient.Client, os.Getenv("MULTUS_NODE_NAME"))
+	netdefInformerFactory, netdefInformer := newNetDefInformer(kubeClient.NetClient)
+	kubeClient.SetK8sClientInformers(podInformer, netdefInformer)
 
 	router := http.NewServeMux()
 	s := &Server{
@@ -174,7 +284,14 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 				[]string{"handler", "code", "method"},
 			),
 		},
+		informerFactory:          informerFactory,
+		podInformer:              podInformer,
+		netdefInformerFactory:    netdefInformerFactory,
+		netdefInformer:           netdefInformer,
+		ignoreReadinessIndicator: ignoreReadinessIndicator,
 	}
+	s.SetKeepAlivesEnabled(false)
+
 	// register metrics
 	prometheus.MustRegister(s.metrics.requestCounter)
 
@@ -188,7 +305,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 
 			result, err := s.handleCNIRequest(r)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				s.writeCNIErrorResponse(w, err)
 				return
 			}
 
@@ -210,7 +327,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 
 			result, err := s.handleDelegateRequest(r)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+				s.writeCNIErrorResponse(w, err)
 				return
 			}
 
@@ -225,7 +342,7 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 	// handle for '/healthz'
 	router.HandleFunc(api.MultusHealthAPIEndpoint, promhttp.InstrumentHandlerCounter(s.metrics.requestCounter.MustCurryWith(prometheus.Labels{"handler": api.MultusHealthAPIEndpoint}),
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if r.Method != http.MethodGet {
+			if r.Method != http.MethodGet && r.Method != http.MethodPost {
 				http.Error(w, fmt.Sprintf("Method not allowed"), http.StatusMethodNotAllowed)
 				return
 			}
@@ -244,6 +361,65 @@ func newCNIServer(rundir string, kubeClient *k8s.ClientInfo, exec invoke.Exec, s
 	return s, nil
 }
 
+// Start starts the server and begins serving on the given listener
+func (s *Server) Start(ctx context.Context, l net.Listener) {
+	s.informerFactory.Start(ctx.Done())
+	s.netdefInformerFactory.Start(ctx.Done())
+
+	// Give the initial sync some time to complete in large clusters, but
+	// don't wait forever
+	waitCtx, waitCancel := context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), s.podInformer.HasSynced) {
+		logging.Errorf("failed to sync pod informer cache")
+	}
+	waitCancel()
+
+	// Give the initial sync some time to complete in large clusters, but
+	// don't wait forever
+	waitCtx, waitCancel = context.WithTimeout(ctx, 20*time.Second)
+	if !cache.WaitForCacheSync(waitCtx.Done(), s.netdefInformer.HasSynced) {
+		logging.Errorf("failed to sync net-attach-def informer cache")
+	}
+	waitCancel()
+
+	go func() {
+		utilwait.UntilWithContext(ctx, func(_ context.Context) {
+			logging.Debugf("open for business")
+			if err := s.Serve(l); err != nil {
+				utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
+			}
+		}, 0)
+	}()
+}
+
+func (s *Server) writeCNIErrorResponse(w http.ResponseWriter, err error) {
+	var cniErr *cnitypes.Error
+	if errors.As(err, &cniErr) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		errBytes, marshalErr := json.Marshal(cniErr)
+		if marshalErr != nil {
+			http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+			return
+		}
+		if _, writeErr := w.Write(errBytes); writeErr != nil {
+			_ = logging.Errorf("Error writing HTTP response: %v", writeErr)
+		}
+		return
+	}
+	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+}
+
+func (s *Server) wrapCNIRequestError(cmdArgs *skel.CmdArgs, err error) error {
+	var cniErr *cnitypes.Error
+	if errors.As(err, &cniErr) {
+		_ = logging.Errorf("%s ERRORED: %v", printCmdArgs(cmdArgs), err)
+		return err
+	}
+	// Prefix error with request information for easier debugging.
+	return fmt.Errorf("%s ERRORED: %v", printCmdArgs(cmdArgs), err)
+}
+
 func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 	var cr api.Request
 	b, err := io.ReadAll(r.Body)
@@ -253,7 +429,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 	if err := json.Unmarshal(b, &cr); err != nil {
 		return nil, err
 	}
-	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	cmdType, cniCmdArgs, err := s.extractCniData(&cr, s.serverConfig)
 	if err != nil {
 		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
 	}
@@ -263,10 +439,9 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) {
 		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
 	}
 
-	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient)
+	result, err := s.HandleCNIRequest(cmdType, k8sArgs, cniCmdArgs)
 	if err != nil {
-		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+		return nil, s.wrapCNIRequestError(cniCmdArgs, err)
 	}
 	return result, nil
 }
@@ -280,7 +455,7 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 	if err := json.Unmarshal(b, &cr); err != nil {
 		return nil, err
 	}
-	cmdType, cniCmdArgs, err := extractCniData(&cr, s.serverConfig)
+	cmdType, cniCmdArgs, err := s.extractCniData(&cr, s.serverConfig)
 	if err != nil {
 		return nil, fmt.Errorf("could not extract the CNI command args: %w", err)
 	}
@@ -290,15 +465,49 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 		return nil, fmt.Errorf("could not extract the kubernetes runtime args: %w", err)
 	}
 
-	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, s.exec, s.kubeclient, cr.InterfaceAttributes)
+	result, err := s.HandleDelegateRequest(cmdType, k8sArgs, cniCmdArgs, cr.InterfaceAttributes)
 	if err != nil {
-		// Prefix error with request information for easier debugging
-		return nil, fmt.Errorf("%+v %v", cniCmdArgs, err)
+		return nil, s.wrapCNIRequestError(cniCmdArgs, err)
 	}
 	return result, nil
 }
 
-func extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
+func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, ignoreReadinessIndicator bool) ([]byte, error) {
+	if len(overrideConf) == 0 {
+		return cniConf, nil
+	}
+
+	var cni map[string]interface{}
+	if err := json.Unmarshal(cniConf, &cni); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall CNI config: %w", err)
+	}
+
+	var override map[string]interface{}
+	if err := json.Unmarshal(overrideConf, &override); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall CNI override config: %w", err)
+	}
+
+	// Copy each key of the override config into the CNI config except for
+	// a few specific keys
+	ignoreKeys := sets.NewString()
+	if ignoreReadinessIndicator {
+		ignoreKeys.Insert("readinessindicatorfile")
+	}
+	for overrideKey, overrideVal := range override {
+		if !ignoreKeys.Has(overrideKey) {
+			cni[overrideKey] = overrideVal
+		}
+	}
+
+	newBytes, err := json.Marshal(cni)
+	if err != nil {
+		return nil, fmt.Errorf("failed ot marshall new CNI config with overrides: %w", err)
+	}
+
+	return newBytes, nil
+}
+
+func (s *Server) extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel.CmdArgs, error) {
 	cmd, ok := cniRequest.Env["CNI_COMMAND"]
 	if !ok {
 		return "", nil, fmt.Errorf("unexpected or missing CNI_COMMAND")
@@ -325,18 +534,10 @@ func extractCniData(cniRequest *api.Request, overrideConf []byte) (string, *skel
 	}
 	cniCmdArgs.Args = cniArgs
 
-	if overrideConf != nil {
-		// trim the close bracket from multus CNI config and put the server config
-		// to override CNI config with server config.
-		// note: if there are two or more value in same key, then the
-		// latest one is used at golang json implementation
-		idx := bytes.LastIndex(cniRequest.Config, []byte("}"))
-		if idx == -1 {
-			return "", nil, fmt.Errorf("invalid CNI config")
-		}
-		cniCmdArgs.StdinData = append(cniRequest.Config[:idx], overrideConf...)
-	} else {
-		cniCmdArgs.StdinData = cniRequest.Config
+	var err error
+	cniCmdArgs.StdinData, err = overrideCNIConfigWithServerConfig(cniRequest.Config, overrideConf, s.ignoreReadinessIndicator)
+	if err != nil {
+		return "", nil, err
 	}
 
 	return cmd, cniCmdArgs, nil
@@ -409,7 +610,7 @@ func podUID(kubeclient *k8s.ClientInfo, cniArgs map[string]string, podNamespace,
 	return uid, nil
 }
 
-func cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) ([]byte, error) {
+func (s *Server) cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) ([]byte, error) {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -417,14 +618,14 @@ func cmdAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kub
 	}
 
 	logging.Debugf("CmdAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	result, err := multus.CmdAdd(cmdArgs, exec, kubeClient)
+	result, err := multus.CmdAdd(cmdArgs, s.exec, s.kubeclient)
 	if err != nil {
 		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
 	}
 	return serializeResult(result)
 }
 
-func cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+func (s *Server) cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -432,10 +633,10 @@ func cmdDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kub
 	}
 
 	logging.Debugf("CmdDel for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	return multus.CmdDel(cmdArgs, exec, kubeClient)
+	return multus.CmdDel(cmdArgs, s.exec, s.kubeclient)
 }
 
-func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) error {
+func (s *Server) cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
@@ -443,7 +644,29 @@ func cmdCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, k
 	}
 
 	logging.Debugf("CmdCheck for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
-	return multus.CmdCheck(cmdArgs, exec, kubeClient)
+	return multus.CmdCheck(cmdArgs, s.exec, s.kubeclient)
+}
+
+func (s *Server) cmdGC(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdGC for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdGC(cmdArgs, s.exec, s.kubeclient)
+}
+
+func (s *Server) cmdStatus(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs) error {
+	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	podName := string(k8sArgs.K8S_POD_NAME)
+	if namespace == "" || podName == "" {
+		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
+	}
+
+	logging.Debugf("CmdStatus for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
+	return multus.CmdStatus(cmdArgs, s.exec, s.kubeclient)
 }
 
 func serializeResult(result cnitypes.Result) ([]byte, error) {
@@ -460,13 +683,13 @@ func serializeResult(result cnitypes.Result) ([]byte, error) {
 	return responseBytes, nil
 }
 
-func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
+func (s *Server) cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf, interfaceAttributes *api.DelegateInterfaceAttributes) ([]byte, error) {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
 		return nil, fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
 	}
-	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	pod, err := multus.GetPod(s.kubeclient, k8sArgs, false)
 	if err != nil {
 		return nil, err
 	}
@@ -493,7 +716,7 @@ func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 
 	logging.Debugf("CmdDelegateAdd for [%s/%s]. CNI conf: %+v", namespace, podName, *cmdArgs)
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	result, err := multus.DelegateAdd(exec, kubeClient, pod, delegateCNIConf, rt, multusConfig)
+	result, err := multus.DelegateAdd(s.exec, s.kubeclient, pod, delegateCNIConf, rt, multusConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error configuring pod [%s/%s] networking: %v", namespace, podName, err)
 	}
@@ -501,26 +724,35 @@ func cmdDelegateAdd(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 	return serializeResult(result)
 }
 
-func cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, _ *k8s.ClientInfo, multusConfig *types.NetConf) error {
+func (s *Server) cmdDelegateCheck(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
 	delegateCNIConf := &types.DelegateNetConf{}
 	if err := json.Unmarshal(cmdArgs.StdinData, delegateCNIConf); err != nil {
 		return err
 	}
 	delegateCNIConf.Bytes = cmdArgs.StdinData
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	return multus.DelegateCheck(exec, delegateCNIConf, rt, multusConfig)
+	return multus.DelegateCheck(s.exec, delegateCNIConf, rt, multusConfig)
+}
+
+func (s *Server) cmdDelegateStatus(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
+	delegateCNIConf, err := types.LoadDelegateNetConf(cmdArgs.StdinData, nil, "", "")
+	if err != nil {
+		return err
+	}
+	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
+	return multus.DelegateStatus(s.exec, delegateCNIConf, rt, multusConfig)
 }
 
 // note: this function may send back error to the client. In cni spec, command DEL should NOT send any error
 // because container deletion follows cni DEL command. But in delegateDel case, container is not removed by
 // this delegateDel, hence we decide to send error message to the request sender.
-func cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo, multusConfig *types.NetConf) error {
+func (s *Server) cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, multusConfig *types.NetConf) error {
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	if namespace == "" || podName == "" {
 		return fmt.Errorf("required CNI variable missing. pod name: %s; pod namespace: %s", podName, namespace)
 	}
-	pod, err := multus.GetPod(kubeClient, k8sArgs, false)
+	pod, err := multus.GetPod(s.kubeclient, k8sArgs, false)
 	if err != nil {
 		return err
 	}
@@ -530,7 +762,7 @@ func cmdDelegateDel(cmdArgs *skel.CmdArgs, k8sArgs *types.K8sArgs, exec invoke.E
 		return err
 	}
 	rt, _ := types.CreateCNIRuntimeConf(cmdArgs, k8sArgs, cmdArgs.IfName, nil, delegateCNIConf)
-	return multus.DelegateDel(exec, pod, delegateCNIConf, rt, multusConfig)
+	return multus.DelegateDel(s.exec, pod, delegateCNIConf, rt, multusConfig)
 }
 
 // LoadDaemonNetConf loads the configuration for the multus daemon

pkg/server/server_suite_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

pkg/server/server_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+// disable dot-imports only for testing
+//revive:disable:dot-imports
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Server", func() {
+	cniConf := []byte(`{
+	"binDir": "/var/lib/cni/bin",
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"cniVersion": "0.3.1",
+	"daemonSocketDir": "/run/multus/socket",
+	"globalNamespaces": "default,openshift-multus,openshift-sriov-network-operator",
+	"logLevel": "verbose",
+	"logToStderr": true,
+	"name": "multus-cni-network",
+	"namespaceIsolation": true,
+	"type": "multus-shim"
+}`)
+
+	serverConf := []byte(`{
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"readinessindicatorfile": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`)
+
+	Context("correctly overrides incoming CNI config with server config", func() {
+		newConf, err := overrideCNIConfigWithServerConfig(cniConf, serverConf, false)
+		Expect(err).ToNot(HaveOccurred())
+
+		// All server options except readinessindicatorfile should exist
+		// in the returned config
+		Expect(newConf).To(MatchJSON(`{
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"name": "multus-cni-network",
+	"type": "multus-shim",
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"readinessindicatorfile": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`))
+	})
+
+	Context("correctly overrides incoming CNI config with server config and ignores readinessindicatorfile", func() {
+		newConf, err := overrideCNIConfigWithServerConfig(cniConf, serverConf, true)
+		Expect(err).ToNot(HaveOccurred())
+
+		// All server options except readinessindicatorfile should exist
+		// in the returned config
+		Expect(newConf).To(MatchJSON(`{
+	"clusterNetwork": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+	"name": "multus-cni-network",
+	"type": "multus-shim",
+	"cniVersion": "0.4.0",
+	"chrootDir": "/hostroot",
+	"logToStderr": false,
+	"logLevel": "debug",
+	"binDir": "/foo/bar",
+	"cniConfigDir": "/host/etc/cni/net.d",
+	"multusConfigFile": "auto",
+	"multusAutoconfigDir": "/host/run/multus/cni/net.d",
+	"namespaceIsolation": false,
+	"globalNamespaces": "other,namespace",
+	"daemonSocketDir": "/somewhere/socket",
+	"socketDir": "/host/run/multus/socket"
+}`))
+	})
+})

pkg/server/thick_cni_test.go

@@ -14,6 +14,8 @@
 
 package server
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"context"
 	"fmt"
@@ -30,8 +32,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
 
@@ -96,20 +96,27 @@ var _ = Describe(suiteName, func() {
 			containerID = "123456789"
 			ifaceName   = "eth0"
 			podName     = "my-little-pod"
+			configPath  = "/tmp/foo.multus.conf"
 		)
 
 		var (
 			cniServer *Server
 			K8sClient *k8s.ClientInfo
 			netns     ns.NetNS
+			ctx       context.Context
+			cancel    context.CancelFunc
 		)
 
 		BeforeEach(func() {
 			var err error
 			K8sClient = fakeK8sClient()
+			// Touch the default network file.
+			os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 
 			Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
-			cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, nil)
+
+			ctx, cancel = context.WithCancel(context.TODO())
+			cniServer, err = startCNIServer(ctx, thickPluginRunDir, K8sClient, nil)
 			Expect(err).NotTo(HaveOccurred())
 
 			netns, err = testutils.NewNS()
@@ -121,6 +128,12 @@ var _ = Describe(suiteName, func() {
 		})
 
 		AfterEach(func() {
+			cancel()
+			// Cleanup default network file.
+			if _, errStat := os.Stat(configPath); errStat == nil {
+				errRemove := os.Remove(configPath)
+				Expect(errRemove).NotTo(HaveOccurred())
+			}
 			unregisterMetrics(cniServer)
 			Expect(cniServer.Close()).To(Succeed())
 			Expect(teardownCNIEnv()).To(Succeed())
@@ -145,12 +158,15 @@ var _ = Describe(suiteName, func() {
 			containerID = "123456789"
 			ifaceName   = "eth0"
 			podName     = "my-little-pod"
+			configPath  = "/tmp/foo.multus.conf"
 		)
 
 		var (
 			cniServer *Server
 			K8sClient *k8s.ClientInfo
 			netns     ns.NetNS
+			ctx       context.Context
+			cancel    context.CancelFunc
 		)
 
 		BeforeEach(func() {
@@ -162,8 +178,12 @@ var _ = Describe(suiteName, func() {
 				"dummy_key2": "dummy_val2"
 			}`
 
+			// Touch the default network file.
+			os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
 			Expect(FilesystemPreRequirements(thickPluginRunDir)).To(Succeed())
-			cniServer, err = startCNIServer(thickPluginRunDir, K8sClient, []byte(dummyServerConfig))
+
+			ctx, cancel = context.WithCancel(context.TODO())
+			cniServer, err = startCNIServer(ctx, thickPluginRunDir, K8sClient, []byte(dummyServerConfig))
 			Expect(err).NotTo(HaveOccurred())
 
 			netns, err = testutils.NewNS()
@@ -175,6 +195,12 @@ var _ = Describe(suiteName, func() {
 		})
 
 		AfterEach(func() {
+			cancel()
+			// Cleanup default network file.
+			if _, errStat := os.Stat(configPath); errStat == nil {
+				errRemove := os.Remove(configPath)
+				Expect(errRemove).NotTo(HaveOccurred())
+			}
 			unregisterMetrics(cniServer)
 			Expect(cniServer.Close()).To(Succeed())
 			Expect(teardownCNIEnv()).To(Succeed())
@@ -200,7 +226,7 @@ func fakeK8sClient() *k8s.ClientInfo {
 	const magicNumber = 10
 	return &k8s.ClientInfo{
 		Client:        fake.NewSimpleClientset(),
-		NetClient:     netfake.NewSimpleClientset().K8sCniCncfIoV1(),
+		NetClient:     netfake.NewSimpleClientset(),
 		EventRecorder: record.NewFakeRecorder(magicNumber),
 	}
 }
@@ -245,10 +271,10 @@ func createFakePod(k8sClient *k8s.ClientInfo, podName string) error {
 	return err
 }
 
-func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) {
+func startCNIServer(ctx context.Context, runDir string, k8sClient *k8s.ClientInfo, servConfig []byte) (*Server, error) {
 	const period = 0
 
-	cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig)
+	cniServer, err := newCNIServer(runDir, k8sClient, &fakeExec{}, servConfig, true)
 	if err != nil {
 		return nil, err
 	}
@@ -258,12 +284,8 @@ func startCNIServer(runDir string, k8sClient *k8s.ClientInfo, servConfig []byte)
 		return nil, fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(runDir), err)
 	}
 
-	cniServer.SetKeepAlivesEnabled(false)
-	go utilwait.Forever(func() {
-		if err := cniServer.Serve(l); err != nil {
-			utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
-		}
-	}, period)
+	cniServer.Start(ctx, l)
+
 	return cniServer, nil
 }
 
@@ -280,7 +302,7 @@ func referenceConfig(thickPluginSocketDir string) string {
         "name": "node-cni-network",
         "type": "multus",
         "daemonSocketDir": "%s",
-        "defaultnetworkfile": "/tmp/foo.multus.conf",
+        "readinessindicatorfile": "/tmp/foo.multus.conf",
         "defaultnetworkwaitseconds": 3,
         "delegates": [{
             "name": "weave1",

pkg/server/types.go

@@ -16,12 +16,17 @@ package server
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/invoke"
 
 	"github.com/prometheus/client_golang/prometheus"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+
+	netdefinformer "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	"k8s.io/client-go/informers/internalinterfaces"
+	"k8s.io/client-go/tools/cache"
 )
 
 const (
@@ -31,6 +36,8 @@ const (
 	DefaultMultusDaemonConfigFile = "/etc/cni/net.d/multus.d/daemon-config.json"
 	// DefaultMultusRunDir specifies default RunDir for multus
 	DefaultMultusRunDir = "/run/multus/"
+	// DefaultCertDuration specifies default duration for certs in per-node-certs config
+	DefaultCertDuration = 10 * time.Minute
 )
 
 // Metrics represents server's metrics.
@@ -42,19 +49,34 @@ type Metrics struct {
 // the CNI shim requests issued when a pod is added / removed.
 type Server struct {
 	http.Server
-	rundir       string
-	kubeclient   *k8sclient.ClientInfo
-	exec         invoke.Exec
-	serverConfig []byte
-	metrics      *Metrics
+	rundir                string
+	kubeclient            *k8sclient.ClientInfo
+	exec                  invoke.Exec
+	serverConfig          []byte
+	metrics               *Metrics
+	informerFactory       internalinterfaces.SharedInformerFactory
+	podInformer           cache.SharedIndexInformer
+	netdefInformerFactory netdefinformer.SharedInformerFactory
+	netdefInformer        cache.SharedIndexInformer
+
+	ignoreReadinessIndicator bool
+}
+
+// PerNodeCertificate for auto certificate generation for per node
+type PerNodeCertificate struct {
+	Enabled             bool   `json:"enabled,omitempty"`
+	BootstrapKubeconfig string `json:"bootstrapKubeconfig,omitempty"`
+	CertDir             string `json:"certDir,omitempty"`
+	CertDuration        string `json:"certDuration,omitempty"`
 }
 
 // ControllerNetConf for the controller cni configuration
 type ControllerNetConf struct {
-	ChrootDir   string `json:"chrootDir,omitempty"`
-	LogFile     string `json:"logFile"`
-	LogLevel    string `json:"logLevel"`
-	LogToStderr bool   `json:"logToStderr,omitempty"`
+	ChrootDir          string              `json:"chrootDir,omitempty"`
+	LogFile            string              `json:"logFile"`
+	LogLevel           string              `json:"logLevel"`
+	LogToStderr        bool                `json:"logToStderr,omitempty"`
+	PerNodeCertificate *PerNodeCertificate `json:"perNodeCertificate,omitempty"`
 
 	MetricsPort *int `json:"metricsPort,omitempty"`
 

pkg/signals/signals.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2024 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package signals provides handling for os signals.
+package signals
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+
+// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned
+// which is canceled on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+func SetupSignalHandler() context.Context {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-c
+		cancel()
+		<-c
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx
+}

pkg/types/conf.go

@@ -20,15 +20,18 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"strings"
-	"sync"
+	"time"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -40,9 +43,6 @@ const (
 	defaultNonIsolatedNamespace   = "default"
 )
 
-// ChrootMutex provides lock to access host filesystem
-var ChrootMutex *sync.Mutex
-
 // LoadDelegateNetConfList reads DelegateNetConf from bytes
 func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error {
 	logging.Debugf("LoadDelegateNetConfList: %s, %v", string(bytes), delegateConf)
@@ -63,6 +63,112 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 	return nil
 }
 
+// ConvertNetworkConfigListToNetConfList converts a libcni.NetworkConfigList to a NetConfList
+func ConvertNetworkConfigListToNetConfList(ncList *libcni.NetworkConfigList) (*types.NetConfList, error) {
+	// Convert Plugins from []*libcni.PluginConfig to []*types.PluginConf
+	var plugins []*types.PluginConf
+	for _, plugin := range ncList.Plugins {
+		plugins = append(plugins, plugin.Network)
+	}
+
+	// Create NetConfList
+	netConfList := &types.NetConfList{
+		CNIVersion:   ncList.CNIVersion,
+		Name:         ncList.Name,
+		DisableCheck: ncList.DisableCheck,
+		DisableGC:    ncList.DisableGC,
+		Plugins:      plugins,
+	}
+
+	return netConfList, nil
+}
+
+// LoadDelegateNetConfFromConfList converts a libcni.NetworkConfigList into a DelegateNetConf structure
+func LoadDelegateNetConfFromConfList(confList *libcni.NetworkConfigList, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
+	var err error
+	logging.Debugf("LoadDelegateNetConfFromConfList: %v, %v, %s", confList, netElement, deviceID)
+
+	// Convert libcni.NetworkConfigList to NetConfList
+	netConfList, err := ConvertNetworkConfigListToNetConfList(confList)
+	if err != nil {
+		return nil, err
+	}
+
+	delegateConf := &DelegateNetConf{
+		Name:                 netConfList.Name,
+		ConfList:             *netConfList,
+		CNINetworkConfigList: *confList,
+		ConfListPlugin:       true,
+	}
+
+	// Convert the plugins back to bytes for consistency
+	pluginsBytes, err := json.Marshal(netConfList)
+	if err != nil {
+		return nil, logging.Errorf("LoadDelegateNetConfFromConfList: error marshaling netConfList: %v", err)
+	}
+	delegateConf.Bytes = pluginsBytes
+
+	if deviceID != "" {
+		pluginsBytes, err = addDeviceIDInConfList(pluginsBytes, deviceID)
+		if err != nil {
+			return nil, logging.Errorf("LoadDelegateNetConfFromConfList: failed to add deviceID in NetConfList bytes: %v", err)
+		}
+		delegateConf.ResourceName = resourceName
+		delegateConf.DeviceID = deviceID
+	}
+
+	if netElement != nil && netElement.CNIArgs != nil {
+		pluginsBytes, err = addCNIArgsInConfList(pluginsBytes, netElement.CNIArgs)
+		if err != nil {
+			return nil, logging.Errorf("LoadDelegateNetConfFromConfList: failed to add cni-args in NetConfList bytes: %v", err)
+		}
+		delegateConf.Bytes = pluginsBytes
+	}
+
+	if netElement != nil {
+		if netElement.Name != "" {
+			// Overwrite CNI config name with net-attach-def name
+			delegateConf.Name = fmt.Sprintf("%s/%s", netElement.Namespace, netElement.Name)
+		}
+		if netElement.InterfaceRequest != "" {
+			delegateConf.IfnameRequest = netElement.InterfaceRequest
+		}
+		if netElement.MacRequest != "" {
+			delegateConf.MacRequest = netElement.MacRequest
+		}
+		if netElement.IPRequest != nil {
+			delegateConf.IPRequest = netElement.IPRequest
+		}
+		if netElement.BandwidthRequest != nil {
+			delegateConf.BandwidthRequest = netElement.BandwidthRequest
+		}
+		if netElement.PortMappingsRequest != nil {
+			delegateConf.PortMappingsRequest = netElement.PortMappingsRequest
+		}
+		if netElement.GatewayRequest != nil {
+			var list []net.IP
+			if delegateConf.GatewayRequest != nil {
+				list = append(*delegateConf.GatewayRequest, *netElement.GatewayRequest...)
+			} else {
+				list = *netElement.GatewayRequest
+			}
+			delegateConf.GatewayRequest = &list
+		}
+		if netElement.InfinibandGUIDRequest != "" {
+			delegateConf.InfinibandGUIDRequest = netElement.InfinibandGUIDRequest
+		}
+		if netElement.DeviceID != "" {
+			if deviceID != "" {
+				logging.Debugf("Warning: Both RuntimeConfig and ResourceMap provide deviceID. Ignoring RuntimeConfig")
+			} else {
+				delegateConf.DeviceID = netElement.DeviceID
+			}
+		}
+	}
+
+	return delegateConf, nil
+}
+
 // LoadDelegateNetConf converts raw CNI JSON into a DelegateNetConf structure
 func LoadDelegateNetConf(bytes []byte, netElement *NetworkSelectionElement, deviceID string, resourceName string) (*DelegateNetConf, error) {
 	var err error
@@ -609,3 +715,37 @@ func CheckSystemNamespaces(namespace string, systemNamespaces []string) bool {
 	}
 	return false
 }
+
+// GetReadinessIndicatorFile waits for readinessIndicatorFile
+func GetReadinessIndicatorFile(readinessIndicatorFileRaw string) error {
+	cleanpath := filepath.Clean(readinessIndicatorFileRaw)
+	readinessIndicatorFile, err := filepath.Abs(cleanpath)
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path of readinessIndicatorFile: %v", err)
+	}
+
+	pollDuration := 1000 * time.Millisecond
+	pollTimeout := 45 * time.Second
+	return utilwait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
+		_, err := os.Stat(readinessIndicatorFile)
+		return err == nil, nil
+	})
+}
+
+// ReadinessIndicatorExistsNow reports if the readiness indicator exists immediately.
+func ReadinessIndicatorExistsNow(readinessIndicatorFileRaw string) (bool, error) {
+	cleanpath := filepath.Clean(readinessIndicatorFileRaw)
+	readinessIndicatorFile, err := filepath.Abs(cleanpath)
+	if err != nil {
+		return false, fmt.Errorf("failed to get absolute path of readinessIndicatorFile: %v", err)
+	}
+
+	_, err = os.Stat(readinessIndicatorFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}

pkg/types/conf_test.go

@@ -15,6 +15,8 @@
 
 package types
 
+// disable dot-imports only for testing
+//revive:disable:dot-imports
 import (
 	"encoding/json"
 	"fmt"
@@ -599,7 +601,7 @@ var _ = Describe("config operations", func() {
 			StdinData: []byte(`{
     "name": "node-cni-network",
     "type": "multus",
-    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "readinessindicatorfile": "/tmp/foo.multus.conf",
     "defaultnetworkwaitseconds": 3,
     "delegates": [{
         "name": "weave1",
@@ -647,7 +649,7 @@ var _ = Describe("config operations", func() {
 			StdinData: []byte(`{
     "name": "node-cni-network",
     "type": "multus",
-    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "readinessindicatorfile": "/tmp/foo.multus.conf",
     "defaultnetworkwaitseconds": 3,
     "delegates": [{
         "name": "weave1",
@@ -702,9 +704,9 @@ var _ = Describe("config operations", func() {
 		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+		delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
-		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+		GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 
 		Expect(err).NotTo(HaveOccurred())
 	})
@@ -735,9 +737,9 @@ var _ = Describe("config operations", func() {
 		delegate, err := LoadDelegateNetConf([]byte(conf), nil, "0000:00:00.0", "")
 		Expect(err).NotTo(HaveOccurred())
 		fmt.Println("result.Version: ", result.Version())
-		delegateNetStatus, err := netutils.CreateNetworkStatus(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
+		delegateNetStatuses, err := netutils.CreateNetworkStatuses(result, delegate.Conf.Name, delegate.MasterPlugin, nil)
 
-		GinkgoT().Logf("delegateNetStatus %+v\n", delegateNetStatus)
+		GinkgoT().Logf("delegateNetStatuses %+v\n", delegateNetStatuses)
 
 		Expect(err).To(HaveOccurred())
 	})

pkg/types/types.go

@@ -18,10 +18,10 @@ package types
 import (
 	"net"
 
-	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
-
+	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/types"
 	cni100 "github.com/containernetworking/cni/pkg/types/100"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -57,6 +57,7 @@ type NetConf struct {
 	NamespaceIsolation       bool     `json:"namespaceIsolation"`
 	RawNonIsolatedNamespaces string   `json:"globalNamespaces"`
 	NonIsolatedNamespaces    []string `json:"-"`
+	AuxiliaryCNIChainName    string   `json:"auxiliaryCNIChainName,omitempty"`
 
 	// Option to set system namespaces (to avoid to add defaultNetworks)
 	SystemNamespaces []string `json:"systemNamespaces"`
@@ -99,6 +100,7 @@ type BandwidthEntry struct {
 type DelegateNetConf struct {
 	Conf                  types.NetConf
 	ConfList              types.NetConfList
+	CNINetworkConfigList  libcni.NetworkConfigList
 	Name                  string
 	IfnameRequest         string          `json:"ifnameRequest,omitempty"`
 	MacRequest            string          `json:"macRequest,omitempty"`

vendor/github.com/Masterminds/semver/v3/.gitignore

@@ -0,0 +1 @@
+_fuzz/

vendor/github.com/Masterminds/semver/v3/.golangci.yml

@@ -0,0 +1,27 @@
+run:
+  deadline: 2m
+
+linters:
+  disable-all: true
+  enable:
+    - misspell
+    - govet
+    - staticcheck
+    - errcheck
+    - unparam
+    - ineffassign
+    - nakedret
+    - gocyclo
+    - dupl
+    - goimports
+    - revive
+    - gosec
+    - gosimple
+    - typecheck
+    - unused
+
+linters-settings:
+  gofmt:
+    simplify: true
+  dupl:
+    threshold: 600

vendor/github.com/Masterminds/semver/v3/CHANGELOG.md

@@ -0,0 +1,268 @@
+# Changelog
+
+## 3.4.0 (2025-06-27)
+
+### Added
+
+- #268: Added property to Constraints to include prereleases for Check and Validate
+
+### Changed
+
+- #263: Updated Go testing for 1.24, 1.23, and 1.22
+- #269: Updated the error message handling for message case and wrapping errors
+- #266: Restore the ability to have leading 0's when parsing with NewVersion.
+  Opt-out of this by setting CoerceNewVersion to false.
+
+### Fixed
+
+- #257: Fixed the CodeQL link (thanks @dmitris)
+- #262: Restored detailed errors when failed to parse with NewVersion. Opt-out
+  of this by setting DetailedNewVersionErrors to false for faster performance.
+- #267: Handle pre-releases for an "and" group if one constraint includes them
+
+## 3.3.1 (2024-11-19)
+
+### Fixed
+
+- #253: Fix for allowing some version that were invalid
+
+## 3.3.0 (2024-08-27)
+
+### Added
+
+- #238: Add LessThanEqual and GreaterThanEqual functions (thanks @grosser)
+- #213: nil version equality checking (thanks @KnutZuidema)
+
+### Changed
+
+- #241: Simplify StrictNewVersion parsing (thanks @grosser)
+- Testing support up through Go 1.23
+- Minimum version set to 1.21 as this is what's tested now
+- Fuzz testing now supports caching
+
+## 3.2.1 (2023-04-10)
+
+### Changed
+
+- #198: Improved testing around pre-release names
+- #200: Improved code scanning with addition of CodeQL
+- #201: Testing now includes Go 1.20. Go 1.17 has been dropped
+- #202: Migrated Fuzz testing to Go built-in Fuzzing. CI runs daily
+- #203: Docs updated for security details
+
+### Fixed
+
+- #199: Fixed issue with range transformations
+
+## 3.2.0 (2022-11-28)
+
+### Added
+
+- #190: Added text marshaling and unmarshaling
+- #167: Added JSON marshalling for constraints (thanks @SimonTheLeg)
+- #173: Implement encoding.TextMarshaler and encoding.TextUnmarshaler on Version (thanks @MarkRosemaker)
+- #179: Added New() version constructor (thanks @kazhuravlev)
+
+### Changed
+
+- #182/#183: Updated CI testing setup
+
+### Fixed
+
+- #186: Fixing issue where validation of constraint section gave false positives
+- #176: Fix constraints check with *-0 (thanks @mtt0)
+- #181: Fixed Caret operator (^) gives unexpected results when the minor version in constraint is 0 (thanks @arshchimni)
+- #161: Fixed godoc (thanks @afirth)
+
+## 3.1.1 (2020-11-23)
+
+### Fixed
+
+- #158: Fixed issue with generated regex operation order that could cause problem
+
+## 3.1.0 (2020-04-15)
+
+### Added
+
+- #131: Add support for serializing/deserializing SQL (thanks @ryancurrah)
+
+### Changed
+
+- #148: More accurate validation messages on constraints
+
+## 3.0.3 (2019-12-13)
+
+### Fixed
+
+- #141: Fixed issue with <= comparison
+
+## 3.0.2 (2019-11-14)
+
+### Fixed
+
+- #134: Fixed broken constraint checking with ^0.0 (thanks @krmichelos)
+
+## 3.0.1 (2019-09-13)
+
+### Fixed
+
+- #125: Fixes issue with module path for v3
+
+## 3.0.0 (2019-09-12)
+
+This is a major release of the semver package which includes API changes. The Go
+API is compatible with ^1. The Go API was not changed because many people are using
+`go get` without Go modules for their applications and API breaking changes cause
+errors which we have or would need to support.
+
+The changes in this release are the handling based on the data passed into the
+functions. These are described in the added and changed sections below.
+
+### Added
+
+- StrictNewVersion function. This is similar to NewVersion but will return an
+  error if the version passed in is not a strict semantic version. For example,
+  1.2.3 would pass but v1.2.3 or 1.2 would fail because they are not strictly
+  speaking semantic versions. This function is faster, performs fewer operations,
+  and uses fewer allocations than NewVersion.
+- Fuzzing has been performed on NewVersion, StrictNewVersion, and NewConstraint.
+  The Makefile contains the operations used. For more information on you can start
+  on Wikipedia at https://en.wikipedia.org/wiki/Fuzzing
+- Now using Go modules
+
+### Changed
+
+- NewVersion has proper prerelease and metadata validation with error messages
+  to signal an issue with either of them
+- ^ now operates using a similar set of rules to npm/js and Rust/Cargo. If the
+  version is >=1 the ^ ranges works the same as v1. For major versions of 0 the
+  rules have changed. The minor version is treated as the stable version unless
+  a patch is specified and then it is equivalent to =. One difference from npm/js
+  is that prereleases there are only to a specific version (e.g. 1.2.3).
+  Prereleases here look over multiple versions and follow semantic version
+  ordering rules. This pattern now follows along with the expected and requested
+  handling of this packaged by numerous users.
+
+## 1.5.0 (2019-09-11)
+
+### Added
+
+- #103: Add basic fuzzing for `NewVersion()` (thanks @jesse-c)
+
+### Changed
+
+- #82: Clarify wildcard meaning in range constraints and update tests for it (thanks @greysteil)
+- #83: Clarify caret operator range for pre-1.0.0 dependencies (thanks @greysteil)
+- #72: Adding docs comment pointing to vert for a cli
+- #71: Update the docs on pre-release comparator handling
+- #89: Test with new go versions (thanks @thedevsaddam)
+- #87: Added $ to ValidPrerelease for better validation (thanks @jeremycarroll)
+
+### Fixed
+
+- #78: Fix unchecked error in example code (thanks @ravron)
+- #70: Fix the handling of pre-releases and the 0.0.0 release edge case
+- #97: Fixed copyright file for proper display on GitHub
+- #107: Fix handling prerelease when sorting alphanum and num
+- #109: Fixed where Validate sometimes returns wrong message on error
+
+## 1.4.2 (2018-04-10)
+
+### Changed
+
+- #72: Updated the docs to point to vert for a console appliaction
+- #71: Update the docs on pre-release comparator handling
+
+### Fixed
+
+- #70: Fix the handling of pre-releases and the 0.0.0 release edge case
+
+## 1.4.1 (2018-04-02)
+
+### Fixed
+
+- Fixed #64: Fix pre-release precedence issue (thanks @uudashr)
+
+## 1.4.0 (2017-10-04)
+
+### Changed
+
+- #61: Update NewVersion to parse ints with a 64bit int size (thanks @zknill)
+
+## 1.3.1 (2017-07-10)
+
+### Fixed
+
+- Fixed #57: number comparisons in prerelease sometimes inaccurate
+
+## 1.3.0 (2017-05-02)
+
+### Added
+
+- #45: Added json (un)marshaling support (thanks @mh-cbon)
+- Stability marker. See https://masterminds.github.io/stability/
+
+### Fixed
+
+- #51: Fix handling of single digit tilde constraint (thanks @dgodd)
+
+### Changed
+
+- #55: The godoc icon moved from png to svg
+
+## 1.2.3 (2017-04-03)
+
+### Fixed
+
+- #46: Fixed 0.x.x and 0.0.x in constraints being treated as *
+
+## Release 1.2.2 (2016-12-13)
+
+### Fixed
+
+- #34: Fixed issue where hyphen range was not working with pre-release parsing.
+
+## Release 1.2.1 (2016-11-28)
+
+### Fixed
+
+- #24: Fixed edge case issue where constraint "> 0" does not handle "0.0.1-alpha"
+  properly.
+
+## Release 1.2.0 (2016-11-04)
+
+### Added
+
+- #20: Added MustParse function for versions (thanks @adamreese)
+- #15: Added increment methods on versions (thanks @mh-cbon)
+
+### Fixed
+
+- Issue #21: Per the SemVer spec (section 9) a pre-release is unstable and
+  might not satisfy the intended compatibility. The change here ignores pre-releases
+  on constraint checks (e.g., ~ or ^) when a pre-release is not part of the
+  constraint. For example, `^1.2.3` will ignore pre-releases while
+  `^1.2.3-alpha` will include them.
+
+## Release 1.1.1 (2016-06-30)
+
+### Changed
+
+- Issue #9: Speed up version comparison performance (thanks @sdboyer)
+- Issue #8: Added benchmarks (thanks @sdboyer)
+- Updated Go Report Card URL to new location
+- Updated Readme to add code snippet formatting (thanks @mh-cbon)
+- Updating tagging to v[SemVer] structure for compatibility with other tools.
+
+## Release 1.1.0 (2016-03-11)
+
+- Issue #2: Implemented validation to provide reasons a versions failed a
+  constraint.
+
+## Release 1.0.1 (2015-12-31)
+
+- Fixed #1: * constraint failing on valid versions.
+
+## Release 1.0.0 (2015-10-20)
+
+- Initial release

vendor/github.com/Masterminds/semver/v3/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (C) 2014-2019, Matt Butcher and Matt Farina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/Masterminds/semver/v3/Makefile

@@ -0,0 +1,31 @@
+GOPATH=$(shell go env GOPATH)
+GOLANGCI_LINT=$(GOPATH)/bin/golangci-lint
+
+.PHONY: lint
+lint: $(GOLANGCI_LINT)
+	@echo "==> Linting codebase"
+	@$(GOLANGCI_LINT) run
+
+.PHONY: test
+test:
+	@echo "==> Running tests"
+	GO111MODULE=on go test -v
+
+.PHONY: test-cover
+test-cover:
+	@echo "==> Running Tests with coverage"
+	GO111MODULE=on go test -cover .
+
+.PHONY: fuzz
+fuzz:
+	@echo "==> Running Fuzz Tests"
+	go env GOCACHE
+	go test -fuzz=FuzzNewVersion -fuzztime=15s .
+	go test -fuzz=FuzzStrictNewVersion -fuzztime=15s .
+	go test -fuzz=FuzzNewConstraint -fuzztime=15s .
+
+$(GOLANGCI_LINT):
+	# Install golangci-lint. The configuration for it is in the .golangci.yml
+	# file in the root of the repository
+	echo ${GOPATH}
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOPATH)/bin v1.56.2

vendor/github.com/Masterminds/semver/v3/README.md

@@ -0,0 +1,274 @@
+# SemVer
+
+The `semver` package provides the ability to work with [Semantic Versions](http://semver.org) in Go. Specifically it provides the ability to:
+
+* Parse semantic versions
+* Sort semantic versions
+* Check if a semantic version fits within a set of constraints
+* Optionally work with a `v` prefix
+
+[![Stability:
+Active](https://masterminds.github.io/stability/active.svg)](https://masterminds.github.io/stability/active.html)
+[![](https://github.com/Masterminds/semver/workflows/Tests/badge.svg)](https://github.com/Masterminds/semver/actions)
+[![GoDoc](https://img.shields.io/static/v1?label=godoc&message=reference&color=blue)](https://pkg.go.dev/github.com/Masterminds/semver/v3)
+[![Go Report Card](https://goreportcard.com/badge/github.com/Masterminds/semver)](https://goreportcard.com/report/github.com/Masterminds/semver)
+
+## Package Versions
+
+Note, import `github.com/Masterminds/semver/v3` to use the latest version.
+
+There are three major versions fo the `semver` package.
+
+* 3.x.x is the stable and active version. This version is focused on constraint
+  compatibility for range handling in other tools from other languages. It has
+  a similar API to the v1 releases. The development of this version is on the master
+  branch. The documentation for this version is below.
+* 2.x was developed primarily for [dep](https://github.com/golang/dep). There are
+  no tagged releases and the development was performed by [@sdboyer](https://github.com/sdboyer).
+  There are API breaking changes from v1. This version lives on the [2.x branch](https://github.com/Masterminds/semver/tree/2.x).
+* 1.x.x is the original release. It is no longer maintained. You should use the
+  v3 release instead. You can read the documentation for the 1.x.x release
+  [here](https://github.com/Masterminds/semver/blob/release-1/README.md).
+
+## Parsing Semantic Versions
+
+There are two functions that can parse semantic versions. The `StrictNewVersion`
+function only parses valid version 2 semantic versions as outlined in the
+specification. The `NewVersion` function attempts to coerce a version into a
+semantic version and parse it. For example, if there is a leading v or a version
+listed without all 3 parts (e.g. `v1.2`) it will attempt to coerce it into a valid
+semantic version (e.g., 1.2.0). In both cases a `Version` object is returned
+that can be sorted, compared, and used in constraints.
+
+When parsing a version an error is returned if there is an issue parsing the
+version. For example,
+
+    v, err := semver.NewVersion("1.2.3-beta.1+build345")
+
+The version object has methods to get the parts of the version, compare it to
+other versions, convert the version back into a string, and get the original
+string. Getting the original string is useful if the semantic version was coerced
+into a valid form.
+
+There are package level variables that affect how `NewVersion` handles parsing.
+
+- `CoerceNewVersion` is `true` by default. When set to `true` it coerces non-compliant
+  versions into SemVer. For example, allowing a leading 0 in a major, minor, or patch
+  part. This enables the use of CalVer in versions even when not compliant with SemVer.
+  When set to `false` less coercion work is done.
+- `DetailedNewVersionErrors` provides more detailed errors. It only has an affect when
+  `CoerceNewVersion` is set to `false`. When `DetailedNewVersionErrors` is set to `true`
+  it can provide some more insight into why a version is invalid. Setting
+  `DetailedNewVersionErrors` to `false` is faster on performance but provides less
+  detailed error messages if a version fails to parse.
+
+## Sorting Semantic Versions
+
+A set of versions can be sorted using the `sort` package from the standard library.
+For example,
+
+```go
+raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",}
+vs := make([]*semver.Version, len(raw))
+for i, r := range raw {
+    v, err := semver.NewVersion(r)
+    if err != nil {
+        t.Errorf("Error parsing version: %s", err)
+    }
+
+    vs[i] = v
+}
+
+sort.Sort(semver.Collection(vs))
+```
+
+## Checking Version Constraints
+
+There are two methods for comparing versions. One uses comparison methods on
+`Version` instances and the other uses `Constraints`. There are some important
+differences to notes between these two methods of comparison.
+
+1. When two versions are compared using functions such as `Compare`, `LessThan`,
+   and others it will follow the specification and always include pre-releases
+   within the comparison. It will provide an answer that is valid with the
+   comparison section of the spec at https://semver.org/#spec-item-11
+2. When constraint checking is used for checks or validation it will follow a
+   different set of rules that are common for ranges with tools like npm/js
+   and Rust/Cargo. This includes considering pre-releases to be invalid if the
+   ranges does not include one. If you want to have it include pre-releases a
+   simple solution is to include `-0` in your range.
+3. Constraint ranges can have some complex rules including the shorthand use of
+   ~ and ^. For more details on those see the options below.
+
+There are differences between the two methods or checking versions because the
+comparison methods on `Version` follow the specification while comparison ranges
+are not part of the specification. Different packages and tools have taken it
+upon themselves to come up with range rules. This has resulted in differences.
+For example, npm/js and Cargo/Rust follow similar patterns while PHP has a
+different pattern for ^. The comparison features in this package follow the
+npm/js and Cargo/Rust lead because applications using it have followed similar
+patters with their versions.
+
+Checking a version against version constraints is one of the most featureful
+parts of the package.
+
+```go
+c, err := semver.NewConstraint(">= 1.2.3")
+if err != nil {
+    // Handle constraint not being parsable.
+}
+
+v, err := semver.NewVersion("1.3")
+if err != nil {
+    // Handle version not being parsable.
+}
+// Check if the version meets the constraints. The variable a will be true.
+a := c.Check(v)
+```
+
+### Basic Comparisons
+
+There are two elements to the comparisons. First, a comparison string is a list
+of space or comma separated AND comparisons. These are then separated by || (OR)
+comparisons. For example, `">= 1.2 < 3.0.0 || >= 4.2.3"` is looking for a
+comparison that's greater than or equal to 1.2 and less than 3.0.0 or is
+greater than or equal to 4.2.3.
+
+The basic comparisons are:
+
+* `=`: equal (aliased to no operator)
+* `!=`: not equal
+* `>`: greater than
+* `<`: less than
+* `>=`: greater than or equal to
+* `<=`: less than or equal to
+
+### Working With Prerelease Versions
+
+Pre-releases, for those not familiar with them, are used for software releases
+prior to stable or generally available releases. Examples of pre-releases include
+development, alpha, beta, and release candidate releases. A pre-release may be
+a version such as `1.2.3-beta.1` while the stable release would be `1.2.3`. In the
+order of precedence, pre-releases come before their associated releases. In this
+example `1.2.3-beta.1 < 1.2.3`.
+
+According to the Semantic Version specification, pre-releases may not be
+API compliant with their release counterpart. It says,
+
+> A pre-release version indicates that the version is unstable and might not satisfy the intended compatibility requirements as denoted by its associated normal version.
+
+SemVer's comparisons using constraints without a pre-release comparator will skip
+pre-release versions. For example, `>=1.2.3` will skip pre-releases when looking
+at a list of releases while `>=1.2.3-0` will evaluate and find pre-releases.
+
+The reason for the `0` as a pre-release version in the example comparison is
+because pre-releases can only contain ASCII alphanumerics and hyphens (along with
+`.` separators), per the spec. Sorting happens in ASCII sort order, again per the
+spec. The lowest character is a `0` in ASCII sort order
+(see an [ASCII Table](http://www.asciitable.com/))
+
+Understanding ASCII sort ordering is important because A-Z comes before a-z. That
+means `>=1.2.3-BETA` will return `1.2.3-alpha`. What you might expect from case
+sensitivity doesn't apply here. This is due to ASCII sort ordering which is what
+the spec specifies.
+
+The `Constraints` instance returned from `semver.NewConstraint()` has a property
+`IncludePrerelease` that, when set to true, will return prerelease versions when calls
+to `Check()` and `Validate()` are made.
+
+### Hyphen Range Comparisons
+
+There are multiple methods to handle ranges and the first is hyphens ranges.
+These look like:
+
+* `1.2 - 1.4.5` which is equivalent to `>= 1.2 <= 1.4.5`
+* `2.3.4 - 4.5` which is equivalent to `>= 2.3.4 <= 4.5`
+
+Note that `1.2-1.4.5` without whitespace is parsed completely differently; it's
+parsed as a single constraint `1.2.0` with _prerelease_ `1.4.5`.
+
+### Wildcards In Comparisons
+
+The `x`, `X`, and `*` characters can be used as a wildcard character. This works
+for all comparison operators. When used on the `=` operator it falls
+back to the patch level comparison (see tilde below). For example,
+
+* `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
+* `>= 1.2.x` is equivalent to `>= 1.2.0`
+* `<= 2.x` is equivalent to `< 3`
+* `*` is equivalent to `>= 0.0.0`
+
+### Tilde Range Comparisons (Patch)
+
+The tilde (`~`) comparison operator is for patch level ranges when a minor
+version is specified and major level changes when the minor number is missing.
+For example,
+
+* `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0`
+* `~1` is equivalent to `>= 1, < 2`
+* `~2.3` is equivalent to `>= 2.3, < 2.4`
+* `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0`
+* `~1.x` is equivalent to `>= 1, < 2`
+
+### Caret Range Comparisons (Major)
+
+The caret (`^`) comparison operator is for major level changes once a stable
+(1.0.0) release has occurred. Prior to a 1.0.0 release the minor versions acts
+as the API stability level. This is useful when comparisons of API versions as a
+major change is API breaking. For example,
+
+* `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0`
+* `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0`
+* `^2.3` is equivalent to `>= 2.3, < 3`
+* `^2.x` is equivalent to `>= 2.0.0, < 3`
+* `^0.2.3` is equivalent to `>=0.2.3 <0.3.0`
+* `^0.2` is equivalent to `>=0.2.0 <0.3.0`
+* `^0.0.3` is equivalent to `>=0.0.3 <0.0.4`
+* `^0.0` is equivalent to `>=0.0.0 <0.1.0`
+* `^0` is equivalent to `>=0.0.0 <1.0.0`
+
+## Validation
+
+In addition to testing a version against a constraint, a version can be validated
+against a constraint. When validation fails a slice of errors containing why a
+version didn't meet the constraint is returned. For example,
+
+```go
+c, err := semver.NewConstraint("<= 1.2.3, >= 1.4")
+if err != nil {
+    // Handle constraint not being parseable.
+}
+
+v, err := semver.NewVersion("1.3")
+if err != nil {
+    // Handle version not being parseable.
+}
+
+// Validate a version against a constraint.
+a, msgs := c.Validate(v)
+// a is false
+for _, m := range msgs {
+    fmt.Println(m)
+
+    // Loops over the errors which would read
+    // "1.3 is greater than 1.2.3"
+    // "1.3 is less than 1.4"
+}
+```
+
+## Contribute
+
+If you find an issue or want to contribute please file an [issue](https://github.com/Masterminds/semver/issues)
+or [create a pull request](https://github.com/Masterminds/semver/pulls).
+
+## Security
+
+Security is an important consideration for this project. The project currently
+uses the following tools to help discover security issues:
+
+* [CodeQL](https://codeql.github.com)
+* [gosec](https://github.com/securego/gosec)
+* Daily Fuzz testing
+
+If you believe you have found a security vulnerability you can privately disclose
+it through the [GitHub security page](https://github.com/Masterminds/semver/security).

vendor/github.com/Masterminds/semver/v3/SECURITY.md

@@ -0,0 +1,19 @@
+# Security Policy
+
+## Supported Versions
+
+The following versions of semver are currently supported:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 3.x     | :white_check_mark: |
+| 2.x     | :x:                |
+| 1.x     | :x:                |
+
+Fixes are only released for the latest minor version in the form of a patch release.
+
+## Reporting a Vulnerability
+
+You can privately disclose a vulnerability through GitHubs
+[private vulnerability reporting](https://github.com/Masterminds/semver/security/advisories)
+mechanism.

vendor/github.com/Masterminds/semver/v3/collection.go

@@ -0,0 +1,24 @@
+package semver
+
+// Collection is a collection of Version instances and implements the sort
+// interface. See the sort package for more details.
+// https://golang.org/pkg/sort/
+type Collection []*Version
+
+// Len returns the length of a collection. The number of Version instances
+// on the slice.
+func (c Collection) Len() int {
+	return len(c)
+}
+
+// Less is needed for the sort interface to compare two Version objects on the
+// slice. If checks if one is less than the other.
+func (c Collection) Less(i, j int) bool {
+	return c[i].LessThan(c[j])
+}
+
+// Swap is needed for the sort interface to replace the Version objects
+// at two different positions in the slice.
+func (c Collection) Swap(i, j int) {
+	c[i], c[j] = c[j], c[i]
+}

vendor/github.com/Masterminds/semver/v3/constraints.go

@@ -0,0 +1,601 @@
+package semver
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// Constraints is one or more constraint that a semantic version can be
+// checked against.
+type Constraints struct {
+	constraints [][]*constraint
+	containsPre []bool
+
+	// IncludePrerelease specifies if pre-releases should be included in
+	// the results. Note, if a constraint range has a prerelease than
+	// prereleases will be included for that AND group even if this is
+	// set to false.
+	IncludePrerelease bool
+}
+
+// NewConstraint returns a Constraints instance that a Version instance can
+// be checked against. If there is a parse error it will be returned.
+func NewConstraint(c string) (*Constraints, error) {
+
+	// Rewrite - ranges into a comparison operation.
+	c = rewriteRange(c)
+
+	ors := strings.Split(c, "||")
+	lenors := len(ors)
+	or := make([][]*constraint, lenors)
+	hasPre := make([]bool, lenors)
+	for k, v := range ors {
+		// Validate the segment
+		if !validConstraintRegex.MatchString(v) {
+			return nil, fmt.Errorf("improper constraint: %s", v)
+		}
+
+		cs := findConstraintRegex.FindAllString(v, -1)
+		if cs == nil {
+			cs = append(cs, v)
+		}
+		result := make([]*constraint, len(cs))
+		for i, s := range cs {
+			pc, err := parseConstraint(s)
+			if err != nil {
+				return nil, err
+			}
+
+			// If one of the constraints has a prerelease record this.
+			// This information is used when checking all in an "and"
+			// group to ensure they all check for prereleases.
+			if pc.con.pre != "" {
+				hasPre[k] = true
+			}
+
+			result[i] = pc
+		}
+		or[k] = result
+	}
+
+	o := &Constraints{
+		constraints: or,
+		containsPre: hasPre,
+	}
+	return o, nil
+}
+
+// Check tests if a version satisfies the constraints.
+func (cs Constraints) Check(v *Version) bool {
+	// TODO(mattfarina): For v4 of this library consolidate the Check and Validate
+	// functions as the underlying functions make that possible now.
+	// loop over the ORs and check the inner ANDs
+	for i, o := range cs.constraints {
+		joy := true
+		for _, c := range o {
+			if check, _ := c.check(v, (cs.IncludePrerelease || cs.containsPre[i])); !check {
+				joy = false
+				break
+			}
+		}
+
+		if joy {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Validate checks if a version satisfies a constraint. If not a slice of
+// reasons for the failure are returned in addition to a bool.
+func (cs Constraints) Validate(v *Version) (bool, []error) {
+	// loop over the ORs and check the inner ANDs
+	var e []error
+
+	// Capture the prerelease message only once. When it happens the first time
+	// this var is marked
+	var prerelesase bool
+	for i, o := range cs.constraints {
+		joy := true
+		for _, c := range o {
+			// Before running the check handle the case there the version is
+			// a prerelease and the check is not searching for prereleases.
+			if !(cs.IncludePrerelease || cs.containsPre[i]) && v.pre != "" {
+				if !prerelesase {
+					em := fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+					e = append(e, em)
+					prerelesase = true
+				}
+				joy = false
+
+			} else {
+
+				if _, err := c.check(v, (cs.IncludePrerelease || cs.containsPre[i])); err != nil {
+					e = append(e, err)
+					joy = false
+				}
+			}
+		}
+
+		if joy {
+			return true, []error{}
+		}
+	}
+
+	return false, e
+}
+
+func (cs Constraints) String() string {
+	buf := make([]string, len(cs.constraints))
+	var tmp bytes.Buffer
+
+	for k, v := range cs.constraints {
+		tmp.Reset()
+		vlen := len(v)
+		for kk, c := range v {
+			tmp.WriteString(c.string())
+
+			// Space separate the AND conditions
+			if vlen > 1 && kk < vlen-1 {
+				tmp.WriteString(" ")
+			}
+		}
+		buf[k] = tmp.String()
+	}
+
+	return strings.Join(buf, " || ")
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (cs *Constraints) UnmarshalText(text []byte) error {
+	temp, err := NewConstraint(string(text))
+	if err != nil {
+		return err
+	}
+
+	*cs = *temp
+
+	return nil
+}
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (cs Constraints) MarshalText() ([]byte, error) {
+	return []byte(cs.String()), nil
+}
+
+var constraintOps map[string]cfunc
+var constraintRegex *regexp.Regexp
+var constraintRangeRegex *regexp.Regexp
+
+// Used to find individual constraints within a multi-constraint string
+var findConstraintRegex *regexp.Regexp
+
+// Used to validate an segment of ANDs is valid
+var validConstraintRegex *regexp.Regexp
+
+const cvRegex string = `v?([0-9|x|X|\*]+)(\.[0-9|x|X|\*]+)?(\.[0-9|x|X|\*]+)?` +
+	`(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` +
+	`(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`
+
+func init() {
+	constraintOps = map[string]cfunc{
+		"":   constraintTildeOrEqual,
+		"=":  constraintTildeOrEqual,
+		"!=": constraintNotEqual,
+		">":  constraintGreaterThan,
+		"<":  constraintLessThan,
+		">=": constraintGreaterThanEqual,
+		"=>": constraintGreaterThanEqual,
+		"<=": constraintLessThanEqual,
+		"=<": constraintLessThanEqual,
+		"~":  constraintTilde,
+		"~>": constraintTilde,
+		"^":  constraintCaret,
+	}
+
+	ops := `=||!=|>|<|>=|=>|<=|=<|~|~>|\^`
+
+	constraintRegex = regexp.MustCompile(fmt.Sprintf(
+		`^\s*(%s)\s*(%s)\s*$`,
+		ops,
+		cvRegex))
+
+	constraintRangeRegex = regexp.MustCompile(fmt.Sprintf(
+		`\s*(%s)\s+-\s+(%s)\s*`,
+		cvRegex, cvRegex))
+
+	findConstraintRegex = regexp.MustCompile(fmt.Sprintf(
+		`(%s)\s*(%s)`,
+		ops,
+		cvRegex))
+
+	// The first time a constraint shows up will look slightly different from
+	// future times it shows up due to a leading space or comma in a given
+	// string.
+	validConstraintRegex = regexp.MustCompile(fmt.Sprintf(
+		`^(\s*(%s)\s*(%s)\s*)((?:\s+|,\s*)(%s)\s*(%s)\s*)*$`,
+		ops,
+		cvRegex,
+		ops,
+		cvRegex))
+}
+
+// An individual constraint
+type constraint struct {
+	// The version used in the constraint check. For example, if a constraint
+	// is '<= 2.0.0' the con a version instance representing 2.0.0.
+	con *Version
+
+	// The original parsed version (e.g., 4.x from != 4.x)
+	orig string
+
+	// The original operator for the constraint
+	origfunc string
+
+	// When an x is used as part of the version (e.g., 1.x)
+	minorDirty bool
+	dirty      bool
+	patchDirty bool
+}
+
+// Check if a version meets the constraint
+func (c *constraint) check(v *Version, includePre bool) (bool, error) {
+	return constraintOps[c.origfunc](v, c, includePre)
+}
+
+// String prints an individual constraint into a string
+func (c *constraint) string() string {
+	return c.origfunc + c.orig
+}
+
+type cfunc func(v *Version, c *constraint, includePre bool) (bool, error)
+
+func parseConstraint(c string) (*constraint, error) {
+	if len(c) > 0 {
+		m := constraintRegex.FindStringSubmatch(c)
+		if m == nil {
+			return nil, fmt.Errorf("improper constraint: %s", c)
+		}
+
+		cs := &constraint{
+			orig:     m[2],
+			origfunc: m[1],
+		}
+
+		ver := m[2]
+		minorDirty := false
+		patchDirty := false
+		dirty := false
+		if isX(m[3]) || m[3] == "" {
+			ver = fmt.Sprintf("0.0.0%s", m[6])
+			dirty = true
+		} else if isX(strings.TrimPrefix(m[4], ".")) || m[4] == "" {
+			minorDirty = true
+			dirty = true
+			ver = fmt.Sprintf("%s.0.0%s", m[3], m[6])
+		} else if isX(strings.TrimPrefix(m[5], ".")) || m[5] == "" {
+			dirty = true
+			patchDirty = true
+			ver = fmt.Sprintf("%s%s.0%s", m[3], m[4], m[6])
+		}
+
+		con, err := NewVersion(ver)
+		if err != nil {
+
+			// The constraintRegex should catch any regex parsing errors. So,
+			// we should never get here.
+			return nil, errors.New("constraint parser error")
+		}
+
+		cs.con = con
+		cs.minorDirty = minorDirty
+		cs.patchDirty = patchDirty
+		cs.dirty = dirty
+
+		return cs, nil
+	}
+
+	// The rest is the special case where an empty string was passed in which
+	// is equivalent to * or >=0.0.0
+	con, err := StrictNewVersion("0.0.0")
+	if err != nil {
+
+		// The constraintRegex should catch any regex parsing errors. So,
+		// we should never get here.
+		return nil, errors.New("constraint parser error")
+	}
+
+	cs := &constraint{
+		con:        con,
+		orig:       c,
+		origfunc:   "",
+		minorDirty: false,
+		patchDirty: false,
+		dirty:      true,
+	}
+	return cs, nil
+}
+
+// Constraint functions
+func constraintNotEqual(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	if c.dirty {
+		if c.con.Major() != v.Major() {
+			return true, nil
+		}
+		if c.con.Minor() != v.Minor() && !c.minorDirty {
+			return true, nil
+		} else if c.minorDirty {
+			return false, fmt.Errorf("%s is equal to %s", v, c.orig)
+		} else if c.con.Patch() != v.Patch() && !c.patchDirty {
+			return true, nil
+		} else if c.patchDirty {
+			// Need to handle prereleases if present
+			if v.Prerelease() != "" || c.con.Prerelease() != "" {
+				eq := comparePrerelease(v.Prerelease(), c.con.Prerelease()) != 0
+				if eq {
+					return true, nil
+				}
+				return false, fmt.Errorf("%s is equal to %s", v, c.orig)
+			}
+			return false, fmt.Errorf("%s is equal to %s", v, c.orig)
+		}
+	}
+
+	eq := v.Equal(c.con)
+	if eq {
+		return false, fmt.Errorf("%s is equal to %s", v, c.orig)
+	}
+
+	return true, nil
+}
+
+func constraintGreaterThan(v *Version, c *constraint, includePre bool) (bool, error) {
+
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	var eq bool
+
+	if !c.dirty {
+		eq = v.Compare(c.con) == 1
+		if eq {
+			return true, nil
+		}
+		return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
+	}
+
+	if v.Major() > c.con.Major() {
+		return true, nil
+	} else if v.Major() < c.con.Major() {
+		return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
+	} else if c.minorDirty {
+		// This is a range case such as >11. When the version is something like
+		// 11.1.0 is it not > 11. For that we would need 12 or higher
+		return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
+	} else if c.patchDirty {
+		// This is for ranges such as >11.1. A version of 11.1.1 is not greater
+		// which one of 11.2.1 is greater
+		eq = v.Minor() > c.con.Minor()
+		if eq {
+			return true, nil
+		}
+		return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
+	}
+
+	// If we have gotten here we are not comparing pre-preleases and can use the
+	// Compare function to accomplish that.
+	eq = v.Compare(c.con) == 1
+	if eq {
+		return true, nil
+	}
+	return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
+}
+
+func constraintLessThan(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	eq := v.Compare(c.con) < 0
+	if eq {
+		return true, nil
+	}
+	return false, fmt.Errorf("%s is greater than or equal to %s", v, c.orig)
+}
+
+func constraintGreaterThanEqual(v *Version, c *constraint, includePre bool) (bool, error) {
+
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	eq := v.Compare(c.con) >= 0
+	if eq {
+		return true, nil
+	}
+	return false, fmt.Errorf("%s is less than %s", v, c.orig)
+}
+
+func constraintLessThanEqual(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	var eq bool
+
+	if !c.dirty {
+		eq = v.Compare(c.con) <= 0
+		if eq {
+			return true, nil
+		}
+		return false, fmt.Errorf("%s is greater than %s", v, c.orig)
+	}
+
+	if v.Major() > c.con.Major() {
+		return false, fmt.Errorf("%s is greater than %s", v, c.orig)
+	} else if v.Major() == c.con.Major() && v.Minor() > c.con.Minor() && !c.minorDirty {
+		return false, fmt.Errorf("%s is greater than %s", v, c.orig)
+	}
+
+	return true, nil
+}
+
+// ~*, ~>* --> >= 0.0.0 (any)
+// ~2, ~2.x, ~2.x.x, ~>2, ~>2.x ~>2.x.x --> >=2.0.0, <3.0.0
+// ~2.0, ~2.0.x, ~>2.0, ~>2.0.x --> >=2.0.0, <2.1.0
+// ~1.2, ~1.2.x, ~>1.2, ~>1.2.x --> >=1.2.0, <1.3.0
+// ~1.2.3, ~>1.2.3 --> >=1.2.3, <1.3.0
+// ~1.2.0, ~>1.2.0 --> >=1.2.0, <1.3.0
+func constraintTilde(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	if v.LessThan(c.con) {
+		return false, fmt.Errorf("%s is less than %s", v, c.orig)
+	}
+
+	// ~0.0.0 is a special case where all constraints are accepted. It's
+	// equivalent to >= 0.0.0.
+	if c.con.Major() == 0 && c.con.Minor() == 0 && c.con.Patch() == 0 &&
+		!c.minorDirty && !c.patchDirty {
+		return true, nil
+	}
+
+	if v.Major() != c.con.Major() {
+		return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
+	}
+
+	if v.Minor() != c.con.Minor() && !c.minorDirty {
+		return false, fmt.Errorf("%s does not have same major and minor version as %s", v, c.orig)
+	}
+
+	return true, nil
+}
+
+// When there is a .x (dirty) status it automatically opts in to ~. Otherwise
+// it's a straight =
+func constraintTildeOrEqual(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	if c.dirty {
+		return constraintTilde(v, c, includePre)
+	}
+
+	eq := v.Equal(c.con)
+	if eq {
+		return true, nil
+	}
+
+	return false, fmt.Errorf("%s is not equal to %s", v, c.orig)
+}
+
+// ^*      -->  (any)
+// ^1.2.3  -->  >=1.2.3 <2.0.0
+// ^1.2    -->  >=1.2.0 <2.0.0
+// ^1      -->  >=1.0.0 <2.0.0
+// ^0.2.3  -->  >=0.2.3 <0.3.0
+// ^0.2    -->  >=0.2.0 <0.3.0
+// ^0.0.3  -->  >=0.0.3 <0.0.4
+// ^0.0    -->  >=0.0.0 <0.1.0
+// ^0      -->  >=0.0.0 <1.0.0
+func constraintCaret(v *Version, c *constraint, includePre bool) (bool, error) {
+	// The existence of prereleases is checked at the group level and passed in.
+	// Exit early if the version has a prerelease but those are to be ignored.
+	if v.Prerelease() != "" && !includePre {
+		return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
+	}
+
+	// This less than handles prereleases
+	if v.LessThan(c.con) {
+		return false, fmt.Errorf("%s is less than %s", v, c.orig)
+	}
+
+	var eq bool
+
+	// ^ when the major > 0 is >=x.y.z < x+1
+	if c.con.Major() > 0 || c.minorDirty {
+
+		// ^ has to be within a major range for > 0. Everything less than was
+		// filtered out with the LessThan call above. This filters out those
+		// that greater but not within the same major range.
+		eq = v.Major() == c.con.Major()
+		if eq {
+			return true, nil
+		}
+		return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
+	}
+
+	// ^ when the major is 0 and minor > 0 is >=0.y.z < 0.y+1
+	if c.con.Major() == 0 && v.Major() > 0 {
+		return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
+	}
+	// If the con Minor is > 0 it is not dirty
+	if c.con.Minor() > 0 || c.patchDirty {
+		eq = v.Minor() == c.con.Minor()
+		if eq {
+			return true, nil
+		}
+		return false, fmt.Errorf("%s does not have same minor version as %s. Expected minor versions to match when constraint major version is 0", v, c.orig)
+	}
+	// ^ when the minor is 0 and minor > 0 is =0.0.z
+	if c.con.Minor() == 0 && v.Minor() > 0 {
+		return false, fmt.Errorf("%s does not have same minor version as %s", v, c.orig)
+	}
+
+	// At this point the major is 0 and the minor is 0 and not dirty. The patch
+	// is not dirty so we need to check if they are equal. If they are not equal
+	eq = c.con.Patch() == v.Patch()
+	if eq {
+		return true, nil
+	}
+	return false, fmt.Errorf("%s does not equal %s. Expect version and constraint to equal when major and minor versions are 0", v, c.orig)
+}
+
+func isX(x string) bool {
+	switch x {
+	case "x", "*", "X":
+		return true
+	default:
+		return false
+	}
+}
+
+func rewriteRange(i string) string {
+	m := constraintRangeRegex.FindAllStringSubmatch(i, -1)
+	if m == nil {
+		return i
+	}
+	o := i
+	for _, v := range m {
+		t := fmt.Sprintf(">= %s, <= %s ", v[1], v[11])
+		o = strings.Replace(o, v[0], t, 1)
+	}
+
+	return o
+}

vendor/github.com/Masterminds/semver/v3/doc.go

@@ -0,0 +1,184 @@
+/*
+Package semver provides the ability to work with Semantic Versions (http://semver.org) in Go.
+
+Specifically it provides the ability to:
+
+  - Parse semantic versions
+  - Sort semantic versions
+  - Check if a semantic version fits within a set of constraints
+  - Optionally work with a `v` prefix
+
+# Parsing Semantic Versions
+
+There are two functions that can parse semantic versions. The `StrictNewVersion`
+function only parses valid version 2 semantic versions as outlined in the
+specification. The `NewVersion` function attempts to coerce a version into a
+semantic version and parse it. For example, if there is a leading v or a version
+listed without all 3 parts (e.g. 1.2) it will attempt to coerce it into a valid
+semantic version (e.g., 1.2.0). In both cases a `Version` object is returned
+that can be sorted, compared, and used in constraints.
+
+When parsing a version an optional error can be returned if there is an issue
+parsing the version. For example,
+
+	v, err := semver.NewVersion("1.2.3-beta.1+b345")
+
+The version object has methods to get the parts of the version, compare it to
+other versions, convert the version back into a string, and get the original
+string. For more details please see the documentation
+at https://godoc.org/github.com/Masterminds/semver.
+
+# Sorting Semantic Versions
+
+A set of versions can be sorted using the `sort` package from the standard library.
+For example,
+
+	    raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",}
+	    vs := make([]*semver.Version, len(raw))
+		for i, r := range raw {
+			v, err := semver.NewVersion(r)
+			if err != nil {
+				t.Errorf("Error parsing version: %s", err)
+			}
+
+			vs[i] = v
+		}
+
+		sort.Sort(semver.Collection(vs))
+
+# Checking Version Constraints and Comparing Versions
+
+There are two methods for comparing versions. One uses comparison methods on
+`Version` instances and the other is using Constraints. There are some important
+differences to notes between these two methods of comparison.
+
+ 1. When two versions are compared using functions such as `Compare`, `LessThan`,
+    and others it will follow the specification and always include prereleases
+    within the comparison. It will provide an answer valid with the comparison
+    spec section at https://semver.org/#spec-item-11
+ 2. When constraint checking is used for checks or validation it will follow a
+    different set of rules that are common for ranges with tools like npm/js
+    and Rust/Cargo. This includes considering prereleases to be invalid if the
+    ranges does not include on. If you want to have it include pre-releases a
+    simple solution is to include `-0` in your range.
+ 3. Constraint ranges can have some complex rules including the shorthard use of
+    ~ and ^. For more details on those see the options below.
+
+There are differences between the two methods or checking versions because the
+comparison methods on `Version` follow the specification while comparison ranges
+are not part of the specification. Different packages and tools have taken it
+upon themselves to come up with range rules. This has resulted in differences.
+For example, npm/js and Cargo/Rust follow similar patterns which PHP has a
+different pattern for ^. The comparison features in this package follow the
+npm/js and Cargo/Rust lead because applications using it have followed similar
+patters with their versions.
+
+Checking a version against version constraints is one of the most featureful
+parts of the package.
+
+	c, err := semver.NewConstraint(">= 1.2.3")
+	if err != nil {
+	    // Handle constraint not being parsable.
+	}
+
+	v, err := semver.NewVersion("1.3")
+	if err != nil {
+	    // Handle version not being parsable.
+	}
+	// Check if the version meets the constraints. The a variable will be true.
+	a := c.Check(v)
+
+# Basic Comparisons
+
+There are two elements to the comparisons. First, a comparison string is a list
+of comma or space separated AND comparisons. These are then separated by || (OR)
+comparisons. For example, `">= 1.2 < 3.0.0 || >= 4.2.3"` is looking for a
+comparison that's greater than or equal to 1.2 and less than 3.0.0 or is
+greater than or equal to 4.2.3. This can also be written as
+`">= 1.2, < 3.0.0 || >= 4.2.3"`
+
+The basic comparisons are:
+
+  - `=`: equal (aliased to no operator)
+  - `!=`: not equal
+  - `>`: greater than
+  - `<`: less than
+  - `>=`: greater than or equal to
+  - `<=`: less than or equal to
+
+# Hyphen Range Comparisons
+
+There are multiple methods to handle ranges and the first is hyphens ranges.
+These look like:
+
+  - `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5`
+  - `2.3.4 - 4.5` which is equivalent to `>= 2.3.4 <= 4.5`
+
+# Wildcards In Comparisons
+
+The `x`, `X`, and `*` characters can be used as a wildcard character. This works
+for all comparison operators. When used on the `=` operator it falls
+back to the tilde operation. For example,
+
+  - `1.2.x` is equivalent to `>= 1.2.0 < 1.3.0`
+  - `>= 1.2.x` is equivalent to `>= 1.2.0`
+  - `<= 2.x` is equivalent to `<= 3`
+  - `*` is equivalent to `>= 0.0.0`
+
+Tilde Range Comparisons (Patch)
+
+The tilde (`~`) comparison operator is for patch level ranges when a minor
+version is specified and major level changes when the minor number is missing.
+For example,
+
+  - `~1.2.3` is equivalent to `>= 1.2.3 < 1.3.0`
+  - `~1` is equivalent to `>= 1, < 2`
+  - `~2.3` is equivalent to `>= 2.3 < 2.4`
+  - `~1.2.x` is equivalent to `>= 1.2.0 < 1.3.0`
+  - `~1.x` is equivalent to `>= 1 < 2`
+
+Caret Range Comparisons (Major)
+
+The caret (`^`) comparison operator is for major level changes once a stable
+(1.0.0) release has occurred. Prior to a 1.0.0 release the minor versions acts
+as the API stability level. This is useful when comparisons of API versions as a
+major change is API breaking. For example,
+
+  - `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0`
+  - `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0`
+  - `^2.3` is equivalent to `>= 2.3, < 3`
+  - `^2.x` is equivalent to `>= 2.0.0, < 3`
+  - `^0.2.3` is equivalent to `>=0.2.3 <0.3.0`
+  - `^0.2` is equivalent to `>=0.2.0 <0.3.0`
+  - `^0.0.3` is equivalent to `>=0.0.3 <0.0.4`
+  - `^0.0` is equivalent to `>=0.0.0 <0.1.0`
+  - `^0` is equivalent to `>=0.0.0 <1.0.0`
+
+# Validation
+
+In addition to testing a version against a constraint, a version can be validated
+against a constraint. When validation fails a slice of errors containing why a
+version didn't meet the constraint is returned. For example,
+
+	c, err := semver.NewConstraint("<= 1.2.3, >= 1.4")
+	if err != nil {
+	    // Handle constraint not being parseable.
+	}
+
+	v, _ := semver.NewVersion("1.3")
+	if err != nil {
+	    // Handle version not being parseable.
+	}
+
+	// Validate a version against a constraint.
+	a, msgs := c.Validate(v)
+	// a is false
+	for _, m := range msgs {
+	    fmt.Println(m)
+
+	    // Loops over the errors which would read
+	    // "1.3 is greater than 1.2.3"
+	    // "1.3 is less than 1.4"
+	}
+*/
+package semver

vendor/github.com/Masterminds/semver/v3/version.go

@@ -0,0 +1,788 @@
+package semver
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// The compiled version of the regex created at init() is cached here so it
+// only needs to be created once.
+var versionRegex *regexp.Regexp
+var looseVersionRegex *regexp.Regexp
+
+// CoerceNewVersion sets if leading 0's are allowd in the version part. Leading 0's are
+// not allowed in a valid semantic version. When set to true, NewVersion will coerce
+// leading 0's into a valid version.
+var CoerceNewVersion = true
+
+// DetailedNewVersionErrors specifies if detailed errors are returned from the NewVersion
+// function. This is used when CoerceNewVersion is set to false. If set to false
+// ErrInvalidSemVer is returned for an invalid version. This does not apply to
+// StrictNewVersion. Setting this function to false returns errors more quickly.
+var DetailedNewVersionErrors = true
+
+var (
+	// ErrInvalidSemVer is returned a version is found to be invalid when
+	// being parsed.
+	ErrInvalidSemVer = errors.New("invalid semantic version")
+
+	// ErrEmptyString is returned when an empty string is passed in for parsing.
+	ErrEmptyString = errors.New("version string empty")
+
+	// ErrInvalidCharacters is returned when invalid characters are found as
+	// part of a version
+	ErrInvalidCharacters = errors.New("invalid characters in version")
+
+	// ErrSegmentStartsZero is returned when a version segment starts with 0.
+	// This is invalid in SemVer.
+	ErrSegmentStartsZero = errors.New("version segment starts with 0")
+
+	// ErrInvalidMetadata is returned when the metadata is an invalid format
+	ErrInvalidMetadata = errors.New("invalid metadata string")
+
+	// ErrInvalidPrerelease is returned when the pre-release is an invalid format
+	ErrInvalidPrerelease = errors.New("invalid prerelease string")
+)
+
+// semVerRegex is the regular expression used to parse a semantic version.
+// This is not the official regex from the semver spec. It has been modified to allow for loose handling
+// where versions like 2.1 are detected.
+const semVerRegex string = `v?(0|[1-9]\d*)(?:\.(0|[1-9]\d*))?(?:\.(0|[1-9]\d*))?` +
+	`(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?` +
+	`(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?`
+
+// looseSemVerRegex is a regular expression that lets invalid semver expressions through
+// with enough detail that certain errors can be checked for.
+const looseSemVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` +
+	`(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` +
+	`(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`
+
+// Version represents a single semantic version.
+type Version struct {
+	major, minor, patch uint64
+	pre                 string
+	metadata            string
+	original            string
+}
+
+func init() {
+	versionRegex = regexp.MustCompile("^" + semVerRegex + "$")
+	looseVersionRegex = regexp.MustCompile("^" + looseSemVerRegex + "$")
+}
+
+const (
+	num     string = "0123456789"
+	allowed string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-" + num
+)
+
+// StrictNewVersion parses a given version and returns an instance of Version or
+// an error if unable to parse the version. Only parses valid semantic versions.
+// Performs checking that can find errors within the version.
+// If you want to coerce a version such as 1 or 1.2 and parse it as the 1.x
+// releases of semver did, use the NewVersion() function.
+func StrictNewVersion(v string) (*Version, error) {
+	// Parsing here does not use RegEx in order to increase performance and reduce
+	// allocations.
+
+	if len(v) == 0 {
+		return nil, ErrEmptyString
+	}
+
+	// Split the parts into [0]major, [1]minor, and [2]patch,prerelease,build
+	parts := strings.SplitN(v, ".", 3)
+	if len(parts) != 3 {
+		return nil, ErrInvalidSemVer
+	}
+
+	sv := &Version{
+		original: v,
+	}
+
+	// Extract build metadata
+	if strings.Contains(parts[2], "+") {
+		extra := strings.SplitN(parts[2], "+", 2)
+		sv.metadata = extra[1]
+		parts[2] = extra[0]
+		if err := validateMetadata(sv.metadata); err != nil {
+			return nil, err
+		}
+	}
+
+	// Extract build prerelease
+	if strings.Contains(parts[2], "-") {
+		extra := strings.SplitN(parts[2], "-", 2)
+		sv.pre = extra[1]
+		parts[2] = extra[0]
+		if err := validatePrerelease(sv.pre); err != nil {
+			return nil, err
+		}
+	}
+
+	// Validate the number segments are valid. This includes only having positive
+	// numbers and no leading 0's.
+	for _, p := range parts {
+		if !containsOnly(p, num) {
+			return nil, ErrInvalidCharacters
+		}
+
+		if len(p) > 1 && p[0] == '0' {
+			return nil, ErrSegmentStartsZero
+		}
+	}
+
+	// Extract major, minor, and patch
+	var err error
+	sv.major, err = strconv.ParseUint(parts[0], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	sv.minor, err = strconv.ParseUint(parts[1], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	sv.patch, err = strconv.ParseUint(parts[2], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	return sv, nil
+}
+
+// NewVersion parses a given version and returns an instance of Version or
+// an error if unable to parse the version. If the version is SemVer-ish it
+// attempts to convert it to SemVer. If you want  to validate it was a strict
+// semantic version at parse time see StrictNewVersion().
+func NewVersion(v string) (*Version, error) {
+	if CoerceNewVersion {
+		return coerceNewVersion(v)
+	}
+	m := versionRegex.FindStringSubmatch(v)
+	if m == nil {
+
+		// Disabling detailed errors is first so that it is in the fast path.
+		if !DetailedNewVersionErrors {
+			return nil, ErrInvalidSemVer
+		}
+
+		// Check for specific errors with the semver string and return a more detailed
+		// error.
+		m = looseVersionRegex.FindStringSubmatch(v)
+		if m == nil {
+			return nil, ErrInvalidSemVer
+		}
+		err := validateVersion(m)
+		if err != nil {
+			return nil, err
+		}
+		return nil, ErrInvalidSemVer
+	}
+
+	sv := &Version{
+		metadata: m[5],
+		pre:      m[4],
+		original: v,
+	}
+
+	var err error
+	sv.major, err = strconv.ParseUint(m[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing version segment: %w", err)
+	}
+
+	if m[2] != "" {
+		sv.minor, err = strconv.ParseUint(m[2], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing version segment: %w", err)
+		}
+	} else {
+		sv.minor = 0
+	}
+
+	if m[3] != "" {
+		sv.patch, err = strconv.ParseUint(m[3], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing version segment: %w", err)
+		}
+	} else {
+		sv.patch = 0
+	}
+
+	// Perform some basic due diligence on the extra parts to ensure they are
+	// valid.
+
+	if sv.pre != "" {
+		if err = validatePrerelease(sv.pre); err != nil {
+			return nil, err
+		}
+	}
+
+	if sv.metadata != "" {
+		if err = validateMetadata(sv.metadata); err != nil {
+			return nil, err
+		}
+	}
+
+	return sv, nil
+}
+
+func coerceNewVersion(v string) (*Version, error) {
+	m := looseVersionRegex.FindStringSubmatch(v)
+	if m == nil {
+		return nil, ErrInvalidSemVer
+	}
+
+	sv := &Version{
+		metadata: m[8],
+		pre:      m[5],
+		original: v,
+	}
+
+	var err error
+	sv.major, err = strconv.ParseUint(m[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing version segment: %w", err)
+	}
+
+	if m[2] != "" {
+		sv.minor, err = strconv.ParseUint(strings.TrimPrefix(m[2], "."), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing version segment: %w", err)
+		}
+	} else {
+		sv.minor = 0
+	}
+
+	if m[3] != "" {
+		sv.patch, err = strconv.ParseUint(strings.TrimPrefix(m[3], "."), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing version segment: %w", err)
+		}
+	} else {
+		sv.patch = 0
+	}
+
+	// Perform some basic due diligence on the extra parts to ensure they are
+	// valid.
+
+	if sv.pre != "" {
+		if err = validatePrerelease(sv.pre); err != nil {
+			return nil, err
+		}
+	}
+
+	if sv.metadata != "" {
+		if err = validateMetadata(sv.metadata); err != nil {
+			return nil, err
+		}
+	}
+
+	return sv, nil
+}
+
+// New creates a new instance of Version with each of the parts passed in as
+// arguments instead of parsing a version string.
+func New(major, minor, patch uint64, pre, metadata string) *Version {
+	v := Version{
+		major:    major,
+		minor:    minor,
+		patch:    patch,
+		pre:      pre,
+		metadata: metadata,
+		original: "",
+	}
+
+	v.original = v.String()
+
+	return &v
+}
+
+// MustParse parses a given version and panics on error.
+func MustParse(v string) *Version {
+	sv, err := NewVersion(v)
+	if err != nil {
+		panic(err)
+	}
+	return sv
+}
+
+// String converts a Version object to a string.
+// Note, if the original version contained a leading v this version will not.
+// See the Original() method to retrieve the original value. Semantic Versions
+// don't contain a leading v per the spec. Instead it's optional on
+// implementation.
+func (v Version) String() string {
+	var buf bytes.Buffer
+
+	fmt.Fprintf(&buf, "%d.%d.%d", v.major, v.minor, v.patch)
+	if v.pre != "" {
+		fmt.Fprintf(&buf, "-%s", v.pre)
+	}
+	if v.metadata != "" {
+		fmt.Fprintf(&buf, "+%s", v.metadata)
+	}
+
+	return buf.String()
+}
+
+// Original returns the original value passed in to be parsed.
+func (v *Version) Original() string {
+	return v.original
+}
+
+// Major returns the major version.
+func (v Version) Major() uint64 {
+	return v.major
+}
+
+// Minor returns the minor version.
+func (v Version) Minor() uint64 {
+	return v.minor
+}
+
+// Patch returns the patch version.
+func (v Version) Patch() uint64 {
+	return v.patch
+}
+
+// Prerelease returns the pre-release version.
+func (v Version) Prerelease() string {
+	return v.pre
+}
+
+// Metadata returns the metadata on the version.
+func (v Version) Metadata() string {
+	return v.metadata
+}
+
+// originalVPrefix returns the original 'v' prefix if any.
+func (v Version) originalVPrefix() string {
+	// Note, only lowercase v is supported as a prefix by the parser.
+	if v.original != "" && v.original[:1] == "v" {
+		return v.original[:1]
+	}
+	return ""
+}
+
+// IncPatch produces the next patch version.
+// If the current version does not have prerelease/metadata information,
+// it unsets metadata and prerelease values, increments patch number.
+// If the current version has any of prerelease or metadata information,
+// it unsets both values and keeps current patch value
+func (v Version) IncPatch() Version {
+	vNext := v
+	// according to http://semver.org/#spec-item-9
+	// Pre-release versions have a lower precedence than the associated normal version.
+	// according to http://semver.org/#spec-item-10
+	// Build metadata SHOULD be ignored when determining version precedence.
+	if v.pre != "" {
+		vNext.metadata = ""
+		vNext.pre = ""
+	} else {
+		vNext.metadata = ""
+		vNext.pre = ""
+		vNext.patch = v.patch + 1
+	}
+	vNext.original = v.originalVPrefix() + "" + vNext.String()
+	return vNext
+}
+
+// IncMinor produces the next minor version.
+// Sets patch to 0.
+// Increments minor number.
+// Unsets metadata.
+// Unsets prerelease status.
+func (v Version) IncMinor() Version {
+	vNext := v
+	vNext.metadata = ""
+	vNext.pre = ""
+	vNext.patch = 0
+	vNext.minor = v.minor + 1
+	vNext.original = v.originalVPrefix() + "" + vNext.String()
+	return vNext
+}
+
+// IncMajor produces the next major version.
+// Sets patch to 0.
+// Sets minor to 0.
+// Increments major number.
+// Unsets metadata.
+// Unsets prerelease status.
+func (v Version) IncMajor() Version {
+	vNext := v
+	vNext.metadata = ""
+	vNext.pre = ""
+	vNext.patch = 0
+	vNext.minor = 0
+	vNext.major = v.major + 1
+	vNext.original = v.originalVPrefix() + "" + vNext.String()
+	return vNext
+}
+
+// SetPrerelease defines the prerelease value.
+// Value must not include the required 'hyphen' prefix.
+func (v Version) SetPrerelease(prerelease string) (Version, error) {
+	vNext := v
+	if len(prerelease) > 0 {
+		if err := validatePrerelease(prerelease); err != nil {
+			return vNext, err
+		}
+	}
+	vNext.pre = prerelease
+	vNext.original = v.originalVPrefix() + "" + vNext.String()
+	return vNext, nil
+}
+
+// SetMetadata defines metadata value.
+// Value must not include the required 'plus' prefix.
+func (v Version) SetMetadata(metadata string) (Version, error) {
+	vNext := v
+	if len(metadata) > 0 {
+		if err := validateMetadata(metadata); err != nil {
+			return vNext, err
+		}
+	}
+	vNext.metadata = metadata
+	vNext.original = v.originalVPrefix() + "" + vNext.String()
+	return vNext, nil
+}
+
+// LessThan tests if one version is less than another one.
+func (v *Version) LessThan(o *Version) bool {
+	return v.Compare(o) < 0
+}
+
+// LessThanEqual tests if one version is less or equal than another one.
+func (v *Version) LessThanEqual(o *Version) bool {
+	return v.Compare(o) <= 0
+}
+
+// GreaterThan tests if one version is greater than another one.
+func (v *Version) GreaterThan(o *Version) bool {
+	return v.Compare(o) > 0
+}
+
+// GreaterThanEqual tests if one version is greater or equal than another one.
+func (v *Version) GreaterThanEqual(o *Version) bool {
+	return v.Compare(o) >= 0
+}
+
+// Equal tests if two versions are equal to each other.
+// Note, versions can be equal with different metadata since metadata
+// is not considered part of the comparable version.
+func (v *Version) Equal(o *Version) bool {
+	if v == o {
+		return true
+	}
+	if v == nil || o == nil {
+		return false
+	}
+	return v.Compare(o) == 0
+}
+
+// Compare compares this version to another one. It returns -1, 0, or 1 if
+// the version smaller, equal, or larger than the other version.
+//
+// Versions are compared by X.Y.Z. Build metadata is ignored. Prerelease is
+// lower than the version without a prerelease. Compare always takes into account
+// prereleases. If you want to work with ranges using typical range syntaxes that
+// skip prereleases if the range is not looking for them use constraints.
+func (v *Version) Compare(o *Version) int {
+	// Compare the major, minor, and patch version for differences. If a
+	// difference is found return the comparison.
+	if d := compareSegment(v.Major(), o.Major()); d != 0 {
+		return d
+	}
+	if d := compareSegment(v.Minor(), o.Minor()); d != 0 {
+		return d
+	}
+	if d := compareSegment(v.Patch(), o.Patch()); d != 0 {
+		return d
+	}
+
+	// At this point the major, minor, and patch versions are the same.
+	ps := v.pre
+	po := o.Prerelease()
+
+	if ps == "" && po == "" {
+		return 0
+	}
+	if ps == "" {
+		return 1
+	}
+	if po == "" {
+		return -1
+	}
+
+	return comparePrerelease(ps, po)
+}
+
+// UnmarshalJSON implements JSON.Unmarshaler interface.
+func (v *Version) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	temp, err := NewVersion(s)
+	if err != nil {
+		return err
+	}
+	v.major = temp.major
+	v.minor = temp.minor
+	v.patch = temp.patch
+	v.pre = temp.pre
+	v.metadata = temp.metadata
+	v.original = temp.original
+	return nil
+}
+
+// MarshalJSON implements JSON.Marshaler interface.
+func (v Version) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.String())
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (v *Version) UnmarshalText(text []byte) error {
+	temp, err := NewVersion(string(text))
+	if err != nil {
+		return err
+	}
+
+	*v = *temp
+
+	return nil
+}
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (v Version) MarshalText() ([]byte, error) {
+	return []byte(v.String()), nil
+}
+
+// Scan implements the SQL.Scanner interface.
+func (v *Version) Scan(value interface{}) error {
+	var s string
+	s, _ = value.(string)
+	temp, err := NewVersion(s)
+	if err != nil {
+		return err
+	}
+	v.major = temp.major
+	v.minor = temp.minor
+	v.patch = temp.patch
+	v.pre = temp.pre
+	v.metadata = temp.metadata
+	v.original = temp.original
+	return nil
+}
+
+// Value implements the Driver.Valuer interface.
+func (v Version) Value() (driver.Value, error) {
+	return v.String(), nil
+}
+
+func compareSegment(v, o uint64) int {
+	if v < o {
+		return -1
+	}
+	if v > o {
+		return 1
+	}
+
+	return 0
+}
+
+func comparePrerelease(v, o string) int {
+	// split the prelease versions by their part. The separator, per the spec,
+	// is a .
+	sparts := strings.Split(v, ".")
+	oparts := strings.Split(o, ".")
+
+	// Find the longer length of the parts to know how many loop iterations to
+	// go through.
+	slen := len(sparts)
+	olen := len(oparts)
+
+	l := slen
+	if olen > slen {
+		l = olen
+	}
+
+	// Iterate over each part of the prereleases to compare the differences.
+	for i := 0; i < l; i++ {
+		// Since the lentgh of the parts can be different we need to create
+		// a placeholder. This is to avoid out of bounds issues.
+		stemp := ""
+		if i < slen {
+			stemp = sparts[i]
+		}
+
+		otemp := ""
+		if i < olen {
+			otemp = oparts[i]
+		}
+
+		d := comparePrePart(stemp, otemp)
+		if d != 0 {
+			return d
+		}
+	}
+
+	// Reaching here means two versions are of equal value but have different
+	// metadata (the part following a +). They are not identical in string form
+	// but the version comparison finds them to be equal.
+	return 0
+}
+
+func comparePrePart(s, o string) int {
+	// Fastpath if they are equal
+	if s == o {
+		return 0
+	}
+
+	// When s or o are empty we can use the other in an attempt to determine
+	// the response.
+	if s == "" {
+		if o != "" {
+			return -1
+		}
+		return 1
+	}
+
+	if o == "" {
+		if s != "" {
+			return 1
+		}
+		return -1
+	}
+
+	// When comparing strings "99" is greater than "103". To handle
+	// cases like this we need to detect numbers and compare them. According
+	// to the semver spec, numbers are always positive. If there is a - at the
+	// start like -99 this is to be evaluated as an alphanum. numbers always
+	// have precedence over alphanum. Parsing as Uints because negative numbers
+	// are ignored.
+
+	oi, n1 := strconv.ParseUint(o, 10, 64)
+	si, n2 := strconv.ParseUint(s, 10, 64)
+
+	// The case where both are strings compare the strings
+	if n1 != nil && n2 != nil {
+		if s > o {
+			return 1
+		}
+		return -1
+	} else if n1 != nil {
+		// o is a string and s is a number
+		return -1
+	} else if n2 != nil {
+		// s is a string and o is a number
+		return 1
+	}
+	// Both are numbers
+	if si > oi {
+		return 1
+	}
+	return -1
+}
+
+// Like strings.ContainsAny but does an only instead of any.
+func containsOnly(s string, comp string) bool {
+	return strings.IndexFunc(s, func(r rune) bool {
+		return !strings.ContainsRune(comp, r)
+	}) == -1
+}
+
+// From the spec, "Identifiers MUST comprise only
+// ASCII alphanumerics and hyphen [0-9A-Za-z-]. Identifiers MUST NOT be empty.
+// Numeric identifiers MUST NOT include leading zeroes.". These segments can
+// be dot separated.
+func validatePrerelease(p string) error {
+	eparts := strings.Split(p, ".")
+	for _, p := range eparts {
+		if p == "" {
+			return ErrInvalidPrerelease
+		} else if containsOnly(p, num) {
+			if len(p) > 1 && p[0] == '0' {
+				return ErrSegmentStartsZero
+			}
+		} else if !containsOnly(p, allowed) {
+			return ErrInvalidPrerelease
+		}
+	}
+
+	return nil
+}
+
+// From the spec, "Build metadata MAY be denoted by
+// appending a plus sign and a series of dot separated identifiers immediately
+// following the patch or pre-release version. Identifiers MUST comprise only
+// ASCII alphanumerics and hyphen [0-9A-Za-z-]. Identifiers MUST NOT be empty."
+func validateMetadata(m string) error {
+	eparts := strings.Split(m, ".")
+	for _, p := range eparts {
+		if p == "" {
+			return ErrInvalidMetadata
+		} else if !containsOnly(p, allowed) {
+			return ErrInvalidMetadata
+		}
+	}
+	return nil
+}
+
+// validateVersion checks for common validation issues but may not catch all errors
+func validateVersion(m []string) error {
+	var err error
+	var v string
+	if m[1] != "" {
+		if len(m[1]) > 1 && m[1][0] == '0' {
+			return ErrSegmentStartsZero
+		}
+		_, err = strconv.ParseUint(m[1], 10, 64)
+		if err != nil {
+			return fmt.Errorf("error parsing version segment: %w", err)
+		}
+	}
+
+	if m[2] != "" {
+		v = strings.TrimPrefix(m[2], ".")
+		if len(v) > 1 && v[0] == '0' {
+			return ErrSegmentStartsZero
+		}
+		_, err = strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("error parsing version segment: %w", err)
+		}
+	}
+
+	if m[3] != "" {
+		v = strings.TrimPrefix(m[3], ".")
+		if len(v) > 1 && v[0] == '0' {
+			return ErrSegmentStartsZero
+		}
+		_, err = strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("error parsing version segment: %w", err)
+		}
+	}
+
+	if m[5] != "" {
+		if err = validatePrerelease(m[5]); err != nil {
+			return err
+		}
+	}
+
+	if m[8] != "" {
+		if err = validateMetadata(m[8]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}