Merge pull request #1341 from dougbtv/net-attach-def-client-v174

Updates net-attach-def client library to v1.7.4

.github/workflows/build.yml

@@ -0,0 +1,24 @@
+on: [push, pull_request]
+name: Build
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.20.x, 1.21.x]
+        goarch: [386, amd64, arm, arm64, ppc64le, s390x]
+        os: [ubuntu-latest] #, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Build
+      env:
+        GOARCH: ${{ matrix.goarch }}
+        GOOS: ${{ matrix.goos }}
+      run: ./hack/build-go.sh

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: "46 8 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ go ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/image-build.yml

@@ -0,0 +1,82 @@
+name: Image build
+on: [pull_request]
+jobs:
+  build-thin:
+    name: Image build thin plugin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
+      - name: Build container image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest
+          file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+      # note: disable sbom/provenance for now (gchr.io does not managed well yet)
+      - name: Build container debug image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+  build-thick:
+    name: Image thick plugin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build container image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-thick
+          file: images/Dockerfile.thick
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+  build-origin:
+    name: Image build/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Download OKD Builder Dockerfile
+        run: curl https://raw.githubusercontent.com/okd-project/images/main/builder/Dockerfile -o images/okd-builder.Dockerfile
+
+      - name: Patch OKD Builder Dockerfile to workaround error
+        run: sed -i -e "s/yum install -y yum-utils/rpm --import \/etc\/pki\/rpm-gpg\/*;yum install -y yum-utils/" images/okd-builder.Dockerfile
+
+      - name: Create root for builder
+        run: mkdir root
+
+      - name: Organically build golang builder image
+        run: docker build -t local/okdbuilder:latest -f images/okd-builder.Dockerfile .
+
+      - name: Organically build Multus origin image
+        run: docker build -t local/multus-cni:latest-origin -f images/Dockerfile.openshift .

.github/workflows/image-push-master.yml

@@ -0,0 +1,116 @@
+name: Image push for master
+on:
+  push:
+    branches:
+      - master
+env:
+  image-push-owner: 'k8snetworkplumbingwg'
+jobs:
+  push-thick:
+    name: Image push thick image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image for thick plugin
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-thick
+            ghcr.io/${{ github.repository }}:snapshot-thick
+          file: images/Dockerfile.thick
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+  push-thin:
+    name: Image push thin image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push thin container image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest
+            ghcr.io/${{ github.repository }}:snapshot
+          file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+      - name: Push thin container debug image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-debug
+            ghcr.io/${{ github.repository }}:snapshot-debug
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+# TODO: need to fix this action
+#  push-origin:
+#    name: Image push/origin
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v4
+#
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#
+#      - name: Login to GitHub Container Registry
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/login-action@v3
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Push container image
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/build-push-action@v5
+#        with:
+#          context: .
+#          push: true
+#          tags: |
+#            ghcr.io/${{ github.repository }}:latest-origin
+#            ghcr.io/${{ github.repository }}:snapshot-origin
+#          file: images/Dockerfile.openshift
+

.github/workflows/image-push-release.yml

@@ -0,0 +1,138 @@
+name: Image push release
+on:
+  push:
+    tags:
+      - v*
+env:
+  image-push-owner: 'k8snetworkplumbingwg'
+jobs:
+  push-thick:
+    name: Image push thick image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          flavor: |
+            latest=false
+
+      - name: Push container image for thick plugin
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-thick
+            ${{ steps.docker_meta.outputs.tags }}-thick
+          file: images/Dockerfile.thick
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+  push-thin:
+    name: Image push thin image/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          flavor: |
+            latest=false
+
+      - name: Push thin container image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable
+            ${{ steps.docker_meta.outputs.tags }}
+          file: images/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+      - name: Push thin container debug image
+        if: ${{ github.repository_owner == env.image-push-owner }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-debug
+            ${{ steps.docker_meta.outputs.tags }}-debug
+          file: images/Dockerfile.debug
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/ppc64le,linux/s390x
+          sbom: false
+          provenance: false
+
+# TODO: need to fix this action
+#  push-origin:
+#    name: Image push/origin
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v4
+#
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#
+#      - name: Login to GitHub Container Registry
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/login-action@v3
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Docker meta
+#        id: docker_meta
+#        uses: crazy-max/ghaction-docker-meta@v1
+#        with:
+#          images: ghcr.io/${{ github.repository }}
+#          tag-latest: false
+#
+#      - name: Push container image
+#        if: github.repository_owner == 'k8snetworkplumbingwg'
+#        uses: docker/build-push-action@v5
+#        with:
+#          context: .
+#          push: true
+#          tags: |
+#            ghcr.io/${{ github.repository }}:stable-origin
+#            ${{ steps.docker_meta.outputs.tags }}-origin
+#          file: images/Dockerfile.openshift

.github/workflows/kind-e2e.yml

@@ -0,0 +1,107 @@
+name: e2e-kind
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - docker-file: images/Dockerfile.thick
+            cni-version: "0.3.1"
+            multus-manifest: multus-daemonset-thick.yml
+          - docker-file: images/Dockerfile
+            cni-version: "0.3.1"
+            multus-manifest: multus-daemonset.yml
+          - docker-file: images/Dockerfile.thick
+            cni-version: "0.4.0"
+            multus-manifest: multus-daemonset-thick.yml
+          - docker-file: images/Dockerfile
+            cni-version: "0.4.0"
+            multus-manifest: multus-daemonset.yml
+          # need to wait kind to support CNI 1.0.0 (now kind 0.11 supports up to 0.4.0)
+#          - docker-file: images/Dockerfile.thick
+#            cni-version: "1.0.0"
+#            multus-manifest: multus-thick-daemonset.yml
+#          - docker-file: images/Dockerfile
+#            cni-version: "1.0.0"
+#            multus-manifest: multus-daemonset.yml
+    env:
+      JOB_NAME: "${{ matrix.cni-version }}-${{ matrix.multus-manifest }}"
+
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Setup j2cli
+        run: |
+          sudo apt-get install -y j2cli
+          echo $(j2 --version)
+
+      - name: Build latest-amd64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: localhost:5000/multus:e2e
+          file: ${{ matrix.docker-file }}
+          platforms: linux/amd64
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: generate yaml files
+        working-directory: ./e2e
+        run: env CNI_VERSION=${{ matrix.cni-version }} ./generate_yamls.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: MULTUS_MANIFEST=${{ matrix.multus-manifest }} MULTUS_DOCKERFILE=none ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test static pod
+        working-directory: ./e2e
+        run: ./test-static-pod.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+#      - name: Test DRA integration
+#        working-directory: ./e2e
+#        run: ./test-dra-integration.sh
+
+      - name: Export kind logs
+        if: always()
+        run: |
+          mkdir -p /tmp/kind/logs
+          kind export logs --loglevel=debug /tmp/kind/logs
+
+      - name: Upload kind logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
+          path: /tmp/kind/logs
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Release binaries
+on:
+  push:
+    tags:
+      - 'v*'
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.19.x
+
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale-issues-prs.yml

@@ -0,0 +1,15 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          days-before-stale: 90
+          days-before-close: 7

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+on: [push, pull_request]
+name: Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.20.x, 1.21.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Run Revive Action by pulling pre-built image
+      uses: docker://morphy/revive-action:v2
+      with:
+        exclude: "./vendor/..."
+
+    - name: Run go fmt
+      run: go fmt ./...
+      #run: diff -u <(echo -n) <(gofmt -d -s .)
+
+    - name: Run go vet
+      run: go vet ./...
+
+    - name: Test
+      run: sudo ./hack/test-go.sh
+
+    - name: Send coverage
+      uses: shogo82148/actions-goveralls@v1
+      with:
+        path-to-profile: coverage.out
+        flag-name: Go-${{ matrix.go }}
+        parallel: true
+
+  # notifies that all test jobs are finished.
+  finish:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true

.gitignore

@@ -1,9 +1,17 @@
 # Binary output dir
 bin/
+e2e/bin/
+e2e/yamls/
+e2e/repos/
 
 # GOPATH created by the build script
 gopath/
 
+# Editor paths
+.swp*
+.swo*
+.idea*
+
 # Test outputs
 *.out
 *.test

.goreleaser.yml

@@ -6,19 +6,53 @@ before:
   hooks:
     - go mod download
 builds:
--  
-  env:
-  - CGO_ENABLED=0
-  main: ./multus/
-  goos:
-  - linux
-  goarch:
-  - 386
-  - amd64
-  - arm
-  - arm64
-archive:
-  wrap_in_directory: true
+  - env:
+      - CGO_ENABLED=0
+    id: multus
+    binary: multus
+    main: ./cmd/multus
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+  - env:
+      - CGO_ENABLED=0
+    id: multus-daemon
+    binary: multus-daemon
+    main: ./cmd/multus-daemon
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+  - env:
+      - CGO_ENABLED=0
+    id: multus-shim
+    binary: multus-shim
+    main: ./cmd/multus-shim
+    goos:
+      - linux
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+      - s390x
+    ldflags:
+      - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date={{ .Date }}
+archives:
+  - wrap_in_directory: true
 checksum:
   name_template: 'checksums.txt'
 snapshot:

.travis.yml

@@ -1,105 +0,0 @@
-language: go
-# see https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments
-# for the detail
-# sudo: requried
-dist: trusty
-
-go:
-  - 1.12.x
-
-env:
-  global:
-    - GO111MODULE=on
-    - REGISTRY_USER=${REGISTRY_USER}
-    - REGISTRY_PASS=${REGISTRY_PASS}
-    - secure: "${REGISTRY_SECURE}"
-  matrix:
-    - TARGET=amd64
-    - TARGET=ppc64le
-
-before_install:
-  - sudo apt-get update -qq
-  - go get github.com/mattn/goveralls
-
-install:
-  - go get -u golang.org/x/lint/golint
-
-before_script:
-  # Make gopath... to run golint/go fmt/go vet
-  # Suppress golint for fixing lint later.
-  - golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
-  - go fmt ./...
-  - go vet ./...
-#  - gocyclo -over 15 ./multus
-
-script:
-  - GOARCH="${TARGET}" ./build
-  - |
-    if [ "${TARGET}" == "amd64" ]; then
-      sudo env PATH=${PATH} ./test.sh
-      goveralls -coverprofile=coverage.out -service=travis-ci
-      mkdir -p ${TRAVIS_BUILD_DIR}/dist
-      tar cvfz ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz --warning=no-file-changed --exclude="dist" .
-      docker build -t nfvpe/multus:latest-amd64 .
-      docker build -t nfvpe/multus:latest-ppc64le -f Dockerfile.ppc64le .
-      docker build -t nfvpe/multus-origin:latest -f Dockerfile.openshift .
-    fi
-
-deploy:
-  # Release on versioned tag (e.g. v1.0)
-  - provider: script
-    #skip_cleanup: true
-    script: curl -sL https://git.io/goreleaser | bash
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TRAVIS_TAG =~ ^v[0-9].*$"
-  # Push images to Dockerhub on tag
-  - provider: script
-    skip_cleanup: true
-    script: >
-      bash -c '
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:latest;
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:stable;
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:stable-amd64;
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:$TRAVIS_TAG;
-      docker tag nfvpe/multus:latest-ppc64le nfvpe/multus:stable-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
-      docker push nfvpe/multus:latest;
-      docker push nfvpe/multus:latest-amd64;
-      docker push nfvpe/multus:latest-ppc64le;
-      docker push nfvpe/multus:stable;
-      docker push nfvpe/multus:stable-amd64;
-      docker push nfvpe/multus:stable-ppc64le;
-      docker push nfvpe/multus:$TRAVIS_TAG;
-      echo done'
-    on:
-      tags: true
-      all_branches: true
-      condition: "$TRAVIS_TAG =~ ^v[0-9].*$"
-  # Push images to Dockerhub on merge to master
-  - provider: script
-    on:
-      branch: master
-    script: >
-      bash -c '
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:snapshot;
-      docker tag nfvpe/multus:latest-amd64 nfvpe/multus:snapshot-amd64;
-      docker tag nfvpe/multus:latest-ppc64le nfvpe/multus:snapshot-ppc64le;
-      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
-      docker push nfvpe/multus:snapshot;
-      docker push nfvpe/multus:snapshot-amd64;
-      docker push nfvpe/multus:snapshot-ppc64le;
-      docker push nfvpe/multus:latest;
-      docker push nfvpe/multus:latest-amd64;
-      docker push nfvpe/multus:latest-ppc64le;
-      echo done'
-
-after_success:
-  # put build tgz to bintray
-  - curl -T ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/content/redhat-nfvpe/multus-cni-crd-snapshots/snapshot/snapshot-${TRAVIS_COMMIT}/multus-cni_amd64-${TRAVIS_COMMIT}.tar.gz
-  # publish uploaded file
-  - curl -X POST -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/content/redhat-nfvpe/multus-cni-crd-snapshots/snapshot/snapshot-${TRAVIS_COMMIT}/publish
-  # put it in bintray download list
-  - sleep 20
-  - "curl -X PUT -H 'Accept: application/json' -H 'Content-type: application/json' -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/file_metadata/redhat-nfvpe/multus-cni-crd-snapshots/multus-cni_amd64-${TRAVIS_COMMIT}.tar.gz -d '{\"list_in_downloads\":true}'"

CODE_OF_CONDUCT.md

@@ -0,0 +1,130 @@
+# Multus CNI Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[The Multus Slack Page](https://intel-corp.herokuapp.com/).
+All complaints will be reviewed and investigated promptly and fairly. Or you
+may specifically contact Doug Smith (dosmith@redhat.com) via email.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
+

Dockerfile

@@ -1,20 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM centos:centos7 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV INSTALL_PKGS "git golang"
-RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
-    curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo && \
-    yum install -y $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    cd /usr/src/multus-cni && \
-    ./build
-
-FROM centos:centos7
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
-WORKDIR /
-
-ADD ./images/entrypoint.sh /
-ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.openshift

@@ -1,20 +0,0 @@
-# This dockerfile is specific to building Multus for OpenShift
-FROM openshift/origin-release:golang-1.10 as builder
-
-ADD . /usr/src/multus-cni
-
-WORKDIR /usr/src/multus-cni
-ENV GO111MODULE=off
-RUN ./build
-
-FROM openshift/origin-base
-RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
-COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
-ADD ./images/entrypoint.sh /
-
-LABEL io.k8s.display-name="Multus CNI" \
-      io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
-      io.openshift.tags="openshift" \
-      maintainer="Doug Smith <dosmith@redhat.com>"
-
-ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.ppc64le

@@ -1,25 +0,0 @@
-# This Dockerfile is used to build the image available on DockerHub
-FROM centos:centos7 as build
-
-# Add everything
-ADD . /usr/src/multus-cni
-
-ENV GOARCH "ppc64le"
-ENV GOOS "linux"
-
-ENV INSTALL_PKGS "git golang"
-RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
-    curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo && \
-    yum install -y $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    cd /usr/src/multus-cni && \
-    ./build
-
-# build ppc container
-FROM ppc64le/centos:latest
-COPY --from=build /usr/src/multus-cni /usr/src/multus-cni
-
-WORKDIR /
-ADD ./images/entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]

NOTICE

@@ -0,0 +1,2 @@
+Copyright 2016 Intel Corporation
+Copyright 2021 Multus Authors

README.md

@@ -1,8 +1,8 @@
 # Multus-CNI
 
-![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png)
+![multus-cni Logo](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/images/Multus.png)
 
-[![Travis CI](https://travis-ci.org/intel/multus-cni.svg?branch=master)](https://travis-ci.org/intel/multus-cni/builds)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/intel/multus-cni/badge.svg)](https://coveralls.io/github/intel/multus-cni)
+[![Build](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml)[![Test](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/k8snetworkplumbingwg/multus-cni)](https://goreportcard.com/report/github.com/k8snetworkplumbingwg/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/k8snetworkplumbingwg/multus-cni/badge.svg)](https://coveralls.io/github/k8snetworkplumbingwg/multus-cni)
 
 Multus CNI enables attaching multiple network interfaces to pods in Kubernetes.
 
@@ -18,34 +18,49 @@ Multus is one of the projects in the [Baremetal Container Experience kit](https:
 
 Here's an illustration of the network interfaces attached to a pod, as provisioned by Multus CNI. The diagram shows the pod with three interfaces: `eth0`, `net0` and `net1`. `eth0` connects kubernetes cluster network to connect with kubernetes server/services (e.g. kubernetes api-server, kubelet and so on). `net0` and `net1` are additional network attachments and connect to other networks by using [other CNI plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) (e.g. vlan/vxlan/ptp).
 
-![multus-pod-image](doc/images/multus-pod-image.svg)
+![multus-pod-image](docs/images/multus-pod-image.svg)
 
 ## Quickstart Installation Guide
 
-The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be creatd with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](doc/quickstart.md).
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugin, refer to our [quick-start guide](docs/quickstart.md).
 
-Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
+To use latest features try command below which applies a daemonset and installs thick Multus using `kubectl`:
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
 
-This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](doc/quickstart.md)
+This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
 
-## Additional installation Options
+## Thin Plugin v.s Thick Plugin
 
-- Install via daemonset using the quick-start guide, above.
-- Download binaries from [release page](https://github.com/intel/multus-cni/releases)
-- By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
-- Or, roll-you-own and build from source
-  - See [Development](doc/development.md)
+With the multus 4.0 release, we introduce a new client/server-style plugin deployment. This new deployment is called ['thick plugin'](docs/thick-plugin.md), in contrast to deployments in previous versions, which is now called a 'thin plugin'. The new thick plugin consists of two binaries, multus-daemon and multus-shim CNI plugin. The 'multus-daemon' will be deployed to all nodes as a local agent and supports additional features, such as metrics, which were not available with the 'thin plugin' deployment before. Due to these additional features, the 'thick plugin' comes with the trade-off of consuming more resources than the 'thin plugin'.
+
+We recommend using the thick plugin in most environments, but if you wish to run the thin plugin, or are in a resource-constrained environment, you may do so with:
+
+```
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
+```
+
+## Additional Installation Options
+
+In addition to the [quick-start guide](docs/quickstart.md), you may:
+
+- Download binaries from [release page](https://github.com/k8snetworkplumbingwg/multus-cni/releases)
+- By Docker image from [GitHub Container Registry](https://github.com/orgs/k8snetworkplumbingwg/packages/container/package/multus-cni)
+- Or, roll-your-own and build from source
+  - See [Development](docs/development.md)
 
 ## Comprehensive Documentation
 
-- [How to use](doc/how-to-use.md)
-- [Configuration](doc/configuration.md)
-- [Development](doc/development.md)
+- [How to use](docs/how-to-use.md)
+- [Quick Start Guide](docs/quickstart.md)
+- [Configuration](docs/configuration.md)
+- [Development and Support Information](docs/development.md)
+- [Thick Plugin](docs/thick-plugin.md)
 
 ## Contact Us
 
-For any questions about Multus CNI, feel free to ask a question in #general in the [Intel-Corp Slack](https://intel-corp.herokuapp.com/), or open up a GitHub issue.
+For any questions about Multus CNI, open up a GitHub issue or feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/).
+
+To be invited, use [this slack invite link](https://join.slack.com/t/npwg-team/shared_invite/zt-1u2vmsn2b-tKdOokdPY73zn9B32JoAOg).

build

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DEST_DIR="bin"
-
-if [ ! -d ${DEST_DIR} ]; then
-	mkdir ${DEST_DIR}
-fi
-
-# Add version/commit/date into binary
-# In case of TravisCI, need to check error code of 'git describe'.
-set +e
-git describe --tags --abbrev=0 > /dev/null 2>&1
-if [ "$?" != "0" ]; then
-	VERSION="master"
-else
-	VERSION=$(git describe --tags --abbrev=0)
-fi
-set -e
-DATE=$(date --iso-8601=seconds)
-COMMIT=$(git rev-parse --verify HEAD)
-LDFLAGS="-X main.version=${VERSION:-master} -X main.commit=${COMMIT} -X main.date=${DATE}"
-export CGO_ENABLED=0
-
-# this if... will be removed when gomodules goes default
-if [ "$GO111MODULE" == "off" ]; then
-	echo "Building plugin without go module"
-	echo "Warning: this will be deprecated in near future so please use go modules!"
-
-	ORG_PATH="github.com/intel"
-	REPO_PATH="${ORG_PATH}/multus-cni"
-
-	if [ ! -h gopath/src/${REPO_PATH} ]; then
-		mkdir -p gopath/src/${ORG_PATH}
-		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
-	fi
-
-	export GO15VENDOREXPERIMENT=1
-	export GOBIN=${PWD}/bin
-	export GOPATH=${PWD}/gopath
-	go install -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/multus
-else
-	# build with go modules
-	export GO111MODULE=on
-
-	echo "Building plugins"
-	go build -o ${DEST_DIR}/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ./multus
-fi

checkpoint/checkpoint.go

@@ -1,110 +0,0 @@
-// Copyright (c) 2018 Intel Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package checkpoint
-
-import (
-	"encoding/json"
-	"io/ioutil"
-
-	"github.com/intel/multus-cni/logging"
-	"github.com/intel/multus-cni/types"
-	v1 "k8s.io/api/core/v1"
-)
-
-const (
-	checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint"
-)
-
-// PodDevicesEntry maps PodUID, resource name and allocated device id
-type PodDevicesEntry struct {
-	PodUID        string
-	ContainerName string
-	ResourceName  string
-	DeviceIDs     []string
-	AllocResp     []byte
-}
-
-type checkpointData struct {
-	PodDeviceEntries  []PodDevicesEntry
-	RegisteredDevices map[string][]string
-}
-
-type checkpointFileData struct {
-	Data     checkpointData
-	Checksum uint64
-}
-
-type checkpoint struct {
-	fileName   string
-	podEntires []PodDevicesEntry
-}
-
-// GetCheckpoint returns an instance of Checkpoint
-func GetCheckpoint() (types.ResourceClient, error) {
-	logging.Debugf("GetCheckpoint(): invoked")
-	return getCheckpoint(checkPointfile)
-}
-
-func getCheckpoint(filePath string) (types.ResourceClient, error) {
-	cp := &checkpoint{fileName: filePath}
-	err := cp.getPodEntries()
-	if err != nil {
-		return nil, err
-	}
-	logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath)
-	return cp, nil
-}
-
-// getPodEntries gets all Pod device allocation entries from checkpoint file
-func (cp *checkpoint) getPodEntries() error {
-
-	cpd := &checkpointFileData{}
-	rawBytes, err := ioutil.ReadFile(cp.fileName)
-	if err != nil {
-		return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err)
-	}
-
-	if err = json.Unmarshal(rawBytes, cpd); err != nil {
-		return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err)
-	}
-
-	cp.podEntires = cpd.Data.PodDeviceEntries
-	logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires)
-	return nil
-}
-
-// GetComputeDeviceMap returns an instance of a map of ResourceInfo
-func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
-	podID := string(pod.UID)
-	resourceMap := make(map[string]*types.ResourceInfo)
-
-	if podID == "" {
-		return nil, logging.Errorf("GetPodResourceMap(): invalid Pod cannot be empty")
-	}
-	for _, pod := range cp.podEntires {
-		if pod.PodUID == podID {
-			entry, ok := resourceMap[pod.ResourceName]
-			if ok {
-				// already exists; append to it
-				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
-			} else {
-				// new entry
-				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
-			}
-		}
-	}
-	return resourceMap, nil
-}

cmd/cert-approver/main.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2023 Network Plumbing Working Group
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is Kubernetes controller which approves CSR submitted by multus.
+// This command is required only if multus runs with per-node certificate.
+package main
+
+// Note: cert-approver should be simple, just approve multus' CSR, hence
+// this go code should not have any dependencies from pkg/, if possible,
+// to keep its code simplicity.
+import (
+	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"os/signal"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/certificate/csr"
+	"k8s.io/client-go/util/workqueue"
+)
+
+// CertController object
+type CertController struct {
+	clientset          kubernetes.Interface
+	queue              workqueue.RateLimitingInterface
+	informer           cache.SharedIndexInformer
+	broadcaster        record.EventBroadcaster
+	recorder           record.EventRecorder
+	commonNamePrefixes string
+}
+
+const (
+	maxDuration                = time.Hour * 24 * 365
+	resyncPeriod time.Duration = time.Second * 3600 // resync every one hour, default is 10 hour
+	maxRetries                 = 5
+)
+
+var (
+	// ControllerName provides controller name
+	ControllerName = "csr-approver"
+	// NamePrefix specifies which name in certification request should be target to approve
+	NamePrefix = "system:multus"
+	// Organization specifies which org in certification request should be target to approve
+	Organization = []string{"system:multus"}
+	// Groups specifies which group in certification request should be target to approve
+	Groups = sets.New[string]("system:nodes", "system:multus", "system:authenticated")
+	// UserPrefixes specifies which name prefix in certification request should be target to approve
+	UserPrefixes = sets.New[string]("system:node", NamePrefix)
+	// Usages specifies which usage in certification request should be target to approve
+	Usages = sets.New[certificatesv1.KeyUsage](
+		certificatesv1.UsageDigitalSignature,
+		certificatesv1.UsageClientAuth)
+)
+
+// NewCertController creates certcontroller
+func NewCertController() (*CertController, error) {
+	var clientset kubernetes.Interface
+	// setup Kubernetes API client
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientset, err = kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	informer := cache.NewSharedIndexInformer(
+		cache.NewListWatchFromClient(
+			clientset.CertificatesV1().RESTClient(),
+			"certificatesigningrequests", corev1.NamespaceAll, fields.Everything()),
+		&certificatesv1.CertificateSigningRequest{},
+		resyncPeriod,
+		nil)
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(klog.Infof)
+	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientset.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "cert-approver"})
+	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+	c := &CertController{
+		clientset:          clientset,
+		informer:           informer,
+		queue:              queue,
+		commonNamePrefixes: NamePrefix,
+		broadcaster:        broadcaster,
+		recorder:           recorder,
+	}
+
+	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			if csr, ok := obj.(*certificatesv1.CertificateSigningRequest); ok {
+				if c.filterCSR(csr) {
+					key, err := cache.MetaNamespaceKeyFunc(obj)
+					if err == nil {
+						queue.Add(key)
+					}
+				}
+			}
+		},
+	})
+
+	return c, nil
+}
+
+// Run starts controller
+func (c *CertController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting cert approver")
+
+	go c.informer.Run(stopCh)
+	if !cache.WaitForCacheSync(stopCh, c.HasSynced) {
+		utilruntime.HandleError(fmt.Errorf("Timed out waiting for caches to sync"))
+		return
+	}
+
+	klog.Info("cert approver synced and ready")
+	wait.Until(c.runWorker, time.Second, stopCh)
+}
+
+// HasSynced is required for the cache.Controller interface.
+func (c *CertController) HasSynced() bool {
+	return c.informer.HasSynced()
+}
+
+// LastSyncResourceVersion is required for the cache.Controller interface.
+func (c *CertController) LastSyncResourceVersion() string {
+	return c.informer.LastSyncResourceVersion()
+}
+
+func (c *CertController) runWorker() {
+	for c.processNextItem() {
+		// continue looping
+	}
+}
+
+func (c *CertController) processNextItem() bool {
+	// Wait until there is a new item in the working queue
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	// Tell the queue that we are done with processing this key. This unblocks the key for other workers
+	// This allows safe parallel processing because two pods with the same key are never processed in
+	// parallel.
+	defer c.queue.Done(key)
+
+	// Invoke the method containing the business logic
+	err := c.processItem(key.(string))
+	// Handle the error if something went wrong during the execution of the business logic
+	c.handleErr(err, key)
+	return true
+
+}
+
+// handleErr checks if an error happened and makes sure we will retry later.
+func (c *CertController) handleErr(err error, key interface{}) {
+	if err == nil {
+		// Forget about the #AddRateLimited history of the key on every successful synchronization.
+		// This ensures that future processing of updates for this key is not delayed because of
+		// an outdated error history.
+		c.queue.Forget(key)
+		return
+	}
+
+	// This controller retries 5 times if something goes wrong. After that, it stops trying.
+	if c.queue.NumRequeues(key) < maxRetries {
+		klog.Infof("Error syncing csr %s: %v", key, err)
+		// Re-enqueue the key rate limited. Based on the rate limiter on the
+		// queue and the re-enqueue history, the key will be processed later again.
+		c.queue.AddRateLimited(key)
+		return
+	}
+
+	c.queue.Forget(key)
+	// Report to an external entity that, even after several retries, we could not successfully process this key
+	utilruntime.HandleError(err)
+	klog.Infof("Dropping csr %q out of the queue: %v", key, err)
+}
+
+func (c *CertController) processItem(key string) error {
+	startTime := time.Now()
+
+	obj, _, err := c.informer.GetIndexer().GetByKey(key)
+	if err != nil {
+		return fmt.Errorf("Error fetching object with key %s from store: %v", key, err)
+	}
+
+	req, _ := obj.(*certificatesv1.CertificateSigningRequest)
+
+	nodeName := "unknown"
+	defer func() {
+		klog.Infof("Finished syncing CSR %s for %s node in %v", req.Name, nodeName, time.Since(startTime))
+	}()
+
+	if len(req.Status.Certificate) > 0 {
+		klog.V(5).Infof("CSR %s is already signed", req.Name)
+		return nil
+	}
+
+	if isApprovedOrDenied(&req.Status) {
+		klog.V(5).Infof("CSR %s is already approved/denied", req.Name)
+		return nil
+	}
+
+	csrPEM, _ := pem.Decode(req.Spec.Request)
+	if csrPEM == nil {
+		return fmt.Errorf("failed to PEM-parse the CSR block in .spec.request: no CSRs were found")
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse the CSR bytes: %v", err)
+	}
+
+	i := strings.LastIndex(req.Spec.Username, ":")
+	if i == -1 || i == len(req.Spec.Username)-1 {
+		return fmt.Errorf("failed to parse the username: %s", req.Spec.Username)
+	}
+
+	ctx := context.Background()
+	prefix := req.Spec.Username[:i]
+	nodeName = req.Spec.Username[i+1:]
+	if !UserPrefixes.Has(prefix) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by an unexpected user: %q", req.Name, req.Spec.Username))
+	}
+
+	if errs := validation.IsDNS1123Subdomain(nodeName); len(errs) != 0 {
+		return c.denyCSR(ctx, req, fmt.Sprintf("extracted node name %q is not a valid DNS subdomain %v", nodeName, errs))
+	}
+
+	if usages := sets.New[certificatesv1.KeyUsage](req.Spec.Usages...); !usages.Equal(Usages) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with unexpected usages: %v", req.Name, usages.UnsortedList()))
+	}
+
+	if !Groups.HasAll(req.Spec.Groups...) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created by a user with unexpected groups: %v", req.Name, req.Spec.Groups))
+	}
+
+	expectedSubject := fmt.Sprintf("%s:%s", c.commonNamePrefixes, nodeName)
+	if x509CSR.Subject.CommonName != expectedSubject {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's commonName to be %q, but it is %q", expectedSubject, x509CSR.Subject.CommonName))
+	}
+
+	if !reflect.DeepEqual(x509CSR.Subject.Organization, Organization) {
+		return c.denyCSR(ctx, req, fmt.Sprintf("expected the CSR's organization to be %v, but it is %v", Organization, x509CSR.Subject.Organization))
+	}
+
+	if req.Spec.ExpirationSeconds == nil {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created without specyfying the expirationSeconds", req.Name))
+	}
+
+	if csr.ExpirationSecondsToDuration(*req.Spec.ExpirationSeconds) > maxDuration {
+		return c.denyCSR(ctx, req, fmt.Sprintf("CSR %q was created with invalid expirationSeconds value: %d", req.Name, *req.Spec.ExpirationSeconds))
+	}
+
+	return c.approveCSR(ctx, req)
+}
+
+// CSR specific functions
+
+func (c *CertController) filterCSR(csr *certificatesv1.CertificateSigningRequest) bool {
+	nsName := types.NamespacedName{Namespace: csr.Namespace, Name: csr.Name}
+	csrPEM, _ := pem.Decode(csr.Spec.Request)
+	if csrPEM == nil {
+		klog.Errorf("Failed to PEM-parse the CSR block in .spec.request: no CSRs were found in %s", nsName)
+		return false
+	}
+
+	x509CSR, err := x509.ParseCertificateRequest(csrPEM.Bytes)
+	if err != nil {
+		klog.Errorf("Failed to parse the CSR .spec.request of %q: %v", nsName, err)
+		return false
+	}
+
+	return strings.HasPrefix(x509CSR.Subject.CommonName, c.commonNamePrefixes) &&
+		csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName
+}
+
+func (c *CertController) approveCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateApproved,
+			Status:  corev1.ConditionTrue,
+			Reason:  "AutoApproved",
+			Message: fmt.Sprintf("Auto-approved CSR %q", csr.Name),
+		})
+
+	c.recorder.Eventf(csr, corev1.EventTypeNormal, "CSRApproved", "CSR %q has been approved by %s", csr.Name, ControllerName)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func (c *CertController) denyCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, message string) error {
+	csr.Status.Conditions = append(csr.Status.Conditions,
+		certificatesv1.CertificateSigningRequestCondition{
+			Type:    certificatesv1.CertificateDenied,
+			Status:  corev1.ConditionTrue,
+			Reason:  "CSRDenied",
+			Message: message,
+		},
+	)
+
+	c.recorder.Eventf(csr, corev1.EventTypeWarning, "CSRDenied", "The CSR %q has been denied by: %s", csr.Name, ControllerName, message)
+	_, err := c.clientset.CertificatesV1().CertificateSigningRequests().Update(ctx, csr, metav1.UpdateOptions{})
+	return err
+}
+
+func isApprovedOrDenied(status *certificatesv1.CertificateSigningRequestStatus) bool {
+	for _, c := range status.Conditions {
+		if c.Type == certificatesv1.CertificateApproved || c.Type == certificatesv1.CertificateDenied {
+			return true
+		}
+	}
+	return false
+}
+
+func main() {
+	klog.Infof("starting cert-approver")
+
+	// Start watching for pod creations
+	certController, err := NewCertController()
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+	go certController.Run(stopCh)
+
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+}

cmd/install_multus/main.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a install tool for multus plugins
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+)
+
+func main() {
+	typeFlag := pflag.StringP("type", "t", "", "specify installer type (thick/thin)")
+	destDir := pflag.StringP("dest-dir", "d", "/host/opt/cni/bin", "destination directory")
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	multusFileName := ""
+	switch *typeFlag {
+	case "thick":
+		multusFileName = "multus-shim"
+	case "thin":
+		multusFileName = "multus"
+	default:
+		fmt.Fprintf(os.Stderr, "--type is missing or --type has invalid value\n")
+		os.Exit(1)
+	}
+
+	err := cmdutils.CopyFileAtomic(fmt.Sprintf("/usr/src/multus-cni/bin/%s", multusFileName), *destDir, fmt.Sprintf("%s.temp", multusFileName), multusFileName)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to copy file %s: %v\n", multusFileName, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("multus %s copy succeeded!\n", multusFileName)
+}

cmd/kubeconfig_generator/main.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary submit CSR for kube controll access for multus thin plugin
+// and generate Kubeconfig
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"text/template"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/k8sclient"
+
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+var kubeConfigTemplate = `apiVersion: v1
+clusters:
+  - cluster:
+      certificate-authority-data: {{.CADATA}}
+      server: {{.K8S_APISERVER}}
+    name: default-cluster
+contexts:
+  - context:
+      cluster: default-cluster
+      namespace: default
+      user: default-auth
+    name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+  - name: default-auth
+    user:
+      client-certificate: {{.CERTDIR}}/multus-client-current.pem
+      client-key: {{.CERTDIR}}/multus-client-current.pem
+`
+
+func main() {
+	certDir := pflag.StringP("certdir", "", "/tmp", "specify cert directory")
+	bootstrapConfig := pflag.StringP("bootstrap-config", "", "/tmp/kubeconfig", "specify bootstrap kubernetes config")
+	kubeconfigPathRaw := pflag.StringP("kubeconfig", "", "/run/multus/kubeconfig", "specify output kubeconfig path")
+	certDurationString := pflag.StringP("cert-duration", "", "10m", "specify certificate duration")
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	kubeconfigPath, err := filepath.Abs(*kubeconfigPathRaw)
+	if err != nil {
+		klog.Fatalf("illegal path %s in kubeconfigPath %s: %v", kubeconfigPath, *kubeconfigPathRaw, err)
+	}
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	// check variables
+	if _, err := os.Stat(*bootstrapConfig); err != nil {
+		klog.Fatalf("failed to read bootstrap config %q", *bootstrapConfig)
+	}
+	st, err := os.Stat(*certDir)
+	if err != nil {
+		klog.Fatalf("failed to find cert directory %q", *certDir)
+	}
+	if !st.IsDir() {
+		klog.Fatalf("cert directory %q is not directory", *certDir)
+	}
+	certDuration, err := time.ParseDuration(*certDurationString)
+	if err != nil {
+		klog.Fatalf("failed to parse duration %q: %v", *certDurationString, err)
+	}
+
+	nodeName := os.Getenv("MULTUS_NODE_NAME")
+	if nodeName == "" {
+		klog.Fatalf("cannot identify node name from MULTUS_NODE_NAME env variables")
+	}
+
+	// retrieve API server from bootstrapConfig()
+	config, err := clientcmd.BuildConfigFromFlags("", *bootstrapConfig)
+	if err != nil {
+		klog.Fatalf("cannot get in-cluster config: %v", err)
+	}
+	apiServer := fmt.Sprintf("%s%s", config.Host, config.APIPath)
+	caData := base64.StdEncoding.EncodeToString(config.CAData)
+
+	// run certManager to create certification
+	if _, err = k8sclient.PerNodeK8sClient(nodeName, *bootstrapConfig, certDuration, *certDir); err != nil {
+		klog.Fatalf("failed to start cert manager: %v", err)
+	}
+
+	fp, err := os.OpenFile(kubeconfigPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		klog.Fatalf("cannot create kubeconfig file %q: %v", kubeconfigPath, err)
+	}
+
+	// render kubeconfig
+	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
+	if err != nil {
+		klog.Fatalf("template parse error: %v", err)
+	}
+	templateData := map[string]string{
+		"CADATA":        caData,
+		"CERTDIR":       *certDir,
+		"K8S_APISERVER": apiServer,
+	}
+	// genearate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		klog.Fatalf("cannot create kubeconfig: %v", err)
+	}
+	if err = fp.Close(); err != nil {
+		klog.Fatalf("cannot save kubeconfig: %v", err)
+	}
+
+	klog.Infof("kubeconfig %q is saved", kubeconfigPath)
+
+	// wait for signal
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigterm
+	klog.Infof("signal received. remove kubeconfig %q and quit.", kubeconfigPath)
+	err = os.Remove(kubeconfigPath)
+	if err != nil {
+		klog.Errorf("failed to remove kubeconfig %q: %v", kubeconfigPath, err)
+	}
+}

cmd/multus-daemon/main.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary works as a server that receives requests from multus-shim
+// CNI plugin and creates network interface for kubernets pods.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"os/user"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+	srv "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func main() {
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// keep in command line option
+	version := flag.Bool("version", false, "Show version")
+
+	configFilePath := flag.String("config", srv.DefaultMultusDaemonConfigFile, "Specify the path to the multus-daemon configuration")
+
+	flag.Parse()
+
+	if *version {
+		fmt.Printf("multus-daemon: %s\n", multus.PrintVersionString())
+		os.Exit(4)
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+
+	daemonConf, err := cniServerConfig(*configFilePath)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	multusConf, err := config.ParseMultusConfig(*configFilePath)
+	if err != nil {
+		logging.Panicf("startMultusDaemon failed to load the multus configuration: %v", err)
+		os.Exit(1)
+	}
+
+	logging.Verbosef("multus-daemon started")
+
+	if multusConf.ReadinessIndicatorFile != "" {
+		// Check readinessindicator file before daemon launch
+		logging.Verbosef("Readiness Indicator file check")
+		if err := types.GetReadinessIndicatorFile(multusConf.ReadinessIndicatorFile); err != nil {
+			_ = logging.Errorf("have you checked that your default network is ready? still waiting for readinessindicatorfile @ %v. pollimmediate error: %v", multusConf.ReadinessIndicatorFile, err)
+			os.Exit(1)
+		}
+		logging.Verbosef("Readiness Indicator file check done!")
+	}
+
+	var configManager *config.Manager
+	var ignoreReadinessIndicator bool
+	if multusConf.MultusConfigFile == "auto" {
+		if multusConf.CNIVersion == "" {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		// Generate multus CNI config from current CNI config
+		configManager, err = config.NewManager(*multusConf)
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+		// ConfigManager watches the readiness indicator file (if configured)
+		// and exits the daemon when that is removed. The CNIServer does
+		// not need to re-do that check every CNI operation
+		ignoreReadinessIndicator = true
+	} else {
+		if err := copyUserProvidedConfig(multusConf.MultusConfigFile, multusConf.CniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", multusConf.MultusConfigFile, err)
+		}
+	}
+
+	if err := startMultusDaemon(ctx, daemonConf, ignoreReadinessIndicator); err != nil {
+		logging.Panicf("failed start the multus thick-plugin listener: %v", err)
+		os.Exit(3)
+	}
+
+	// Wait until daemon ready
+	logging.Verbosef("API readiness check")
+	if api.WaitUntilAPIReady(daemonConf.SocketDir) != nil {
+		logging.Panicf("failed to ready multus-daemon socket: %v", err)
+		os.Exit(1)
+	}
+	logging.Verbosef("API readiness check done!")
+
+	signalCh := make(chan os.Signal, 16)
+	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		for sig := range signalCh {
+			logging.Verbosef("caught %v, stopping...", sig)
+			cancel()
+		}
+	}()
+
+	var wg sync.WaitGroup
+	if configManager != nil {
+		if err := configManager.Start(ctx, &wg); err != nil {
+			_ = logging.Errorf("failed to start config manager: %v", err)
+			os.Exit(3)
+		}
+	}
+
+	wg.Wait()
+	logging.Verbosef("multus daemon is exited")
+}
+
+func startMultusDaemon(ctx context.Context, daemonConfig *srv.ControllerNetConf, ignoreReadinessIndicator bool) error {
+	if user, err := user.Current(); err != nil || user.Uid != "0" {
+		return fmt.Errorf("failed to run multus-daemon with root: %v, now running in uid: %s", err, user.Uid)
+	}
+
+	if err := srv.FilesystemPreRequirements(daemonConfig.SocketDir); err != nil {
+		return fmt.Errorf("failed to prepare the cni-socket for communicating with the shim: %w", err)
+	}
+
+	server, err := srv.NewCNIServer(daemonConfig, daemonConfig.ConfigFileContents, ignoreReadinessIndicator)
+	if err != nil {
+		return fmt.Errorf("failed to create the server: %v", err)
+	}
+
+	if daemonConfig.MetricsPort != nil {
+		go utilwait.UntilWithContext(ctx, func(_ context.Context) {
+			http.Handle("/metrics", promhttp.Handler())
+			logging.Debugf("metrics port: %d", *daemonConfig.MetricsPort)
+			logging.Debugf("metrics: %s", http.ListenAndServe(fmt.Sprintf(":%d", *daemonConfig.MetricsPort), nil))
+		}, 0)
+	}
+
+	l, err := srv.GetListener(api.SocketPath(daemonConfig.SocketDir))
+	if err != nil {
+		return fmt.Errorf("failed to start the CNI server using socket %s. Reason: %+v", api.SocketPath(daemonConfig.SocketDir), err)
+	}
+
+	server.Start(ctx, l)
+
+	go func() {
+		<-ctx.Done()
+		server.Shutdown(context.Background())
+	}()
+
+	return nil
+}
+
+func cniServerConfig(configFilePath string) (*srv.ControllerNetConf, error) {
+	path, err := filepath.Abs(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("illegal path %s in server config path %s: %w", path, configFilePath, err)
+	}
+
+	configFileContents, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return srv.LoadDaemonNetConf(configFileContents)
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	path, err := filepath.Abs(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("illegal path %s in multusConfigPath %s: %w", path, multusConfigPath, err)
+	}
+
+	srcFile, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", path, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(dstFile, srcFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/multus-shim/main.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept referred from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/server/api"
+)
+
+func main() {
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+
+	flag.Parse()
+	if versionOpt {
+		fmt.Printf("multus-shim: %s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			return api.CmdAdd(args)
+		},
+		func(args *skel.CmdArgs) error {
+			return api.CmdCheck(args)
+		},
+		func(args *skel.CmdArgs) error {
+			return api.CmdDel(args)
+		},
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

cmd/multus/main.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2016 Intel Corporation
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept referred from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus"
+)
+
+func main() {
+
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt {
+		fmt.Printf("multus: %s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			result, err := multus.CmdAdd(args, nil, nil)
+			if err != nil {
+				return err
+			}
+			return result.Print()
+		},
+		func(args *skel.CmdArgs) error {
+			return multus.CmdCheck(args, nil, nil)
+		},
+		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

cmd/thin_entrypoint/main.go

@@ -0,0 +1,683 @@
+// Copyright (c) 2023 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a entrypoint for thin (stand-alone) images.
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	b64 "encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/containernetworking/cni/libcni"
+	"github.com/spf13/pflag"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
+)
+
+// Options stores command line options
+type Options struct {
+	CNIBinDir                string
+	CNIConfDir               string
+	CNIVersion               string
+	MultusConfFile           string
+	MultusBinFile            string // may be hidden or remove?
+	MultusCNIConfDir         string
+	SkipMultusBinaryCopy     bool
+	MultusKubeConfigFileHost string
+	MultusMasterCNIFileName  string
+	NamespaceIsolation       bool
+	GlobalNamespaces         string
+	MultusAutoconfigDir      string
+	MultusLogToStderr        bool
+	MultusLogLevel           string
+	MultusLogFile            string
+	OverrideNetworkName      bool
+	CleanupConfigOnExit      bool
+	RenameConfFile           bool
+	ReadinessIndicatorFile   string
+	AdditionalBinDir         string
+	ForceCNIVersion          bool
+	SkipTLSVerify            bool
+	SkipMultusConfWatch      bool
+}
+
+const (
+	serviceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+	serviceAccountCAFile    = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+)
+
+func (o *Options) addFlags() {
+	pflag.ErrHelp = nil // suppress error message for help
+	fs := pflag.CommandLine
+	fs.StringVar(&o.CNIBinDir, "cni-bin-dir", "/host/opt/cni/bin", "CNI binary directory")
+	fs.StringVar(&o.CNIConfDir, "cni-conf-dir", "/host/etc/cni/net.d", "CNI config directory")
+	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
+	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
+	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
+	fs.StringVar(&o.MultusCNIConfDir, "multus-cni-conf-dir", "/host/etc/cni/multus/net.d", "multus specific CNI config directory")
+	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")
+
+	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.MultusMasterCNIFileName, "multus-master-cni-file-name", "", "master CNI file in multus-autoconfig-dir")
+	fs.BoolVar(&o.NamespaceIsolation, "namespace-isolation", false, "namespace isolation")
+	fs.StringVar(&o.GlobalNamespaces, "global-namespaces", "", "global namespaces, comma separated (used only with --namespace-isolation=true)")
+	fs.StringVar(&o.MultusAutoconfigDir, "multus-autoconfig-dir", "/host/etc/cni/net.d", "multus autoconfig dir (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.MultusLogToStderr, "multus-log-to-stderr", true, "log to stderr")
+	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
+	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
+	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
+	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
+	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.SkipTLSVerify, "skip-tls-verify", false, "skip TLS verify")
+	fs.BoolVar(&o.ForceCNIVersion, "force-cni-version", false, "force cni version to '--cni-version' (only for e2e-kind testing)")
+	fs.MarkHidden("force-cni-version")
+	fs.MarkHidden("skip-tls-verify")
+}
+
+func (o *Options) verifyFileExists() error {
+	// CNIConfDir
+	if _, err := os.Stat(o.CNIConfDir); err != nil {
+		return fmt.Errorf("cni-conf-dir is not found: %v", err)
+	}
+
+	// CNIBinDir
+	if _, err := os.Stat(o.CNIBinDir); err != nil {
+		return fmt.Errorf("cni-bin-dir is not found: %v", err)
+	}
+
+	// MultusBinFile
+	if _, err := os.Stat(o.MultusBinFile); err != nil {
+		return fmt.Errorf("multus-bin-file is not found: %v", err)
+	}
+
+	if o.MultusConfFile != "auto" {
+		// MultusConfFile
+		if _, err := os.Stat(o.MultusConfFile); err != nil {
+			return fmt.Errorf("multus-conf-file is not found: %v", err)
+		}
+	}
+	return nil
+}
+
+const kubeConfigTemplate = `# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: {{ .KubeConfigHost }}
+    {{ .KubeServerTLS }}
+users:
+- name: multus
+  user:
+    token: "{{ .KubeServiceAccountToken }}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+
+func getFileAndHash(filepath string) ([]byte, []byte, error) {
+	if _, err := os.Stat(filepath); err != nil {
+		return nil, nil, fmt.Errorf("file %s not found: %v", filepath, err)
+	}
+	content, err := os.ReadFile(filepath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot read %s file: %v", filepath, err)
+	}
+
+	hash := sha256.New()
+	hash.Write(content)
+	return content, hash.Sum(nil), nil
+}
+
+func (o *Options) createKubeConfig(prevCAHash, prevSATokenHash []byte) ([]byte, []byte, error) {
+	caFileByte, caHash, err := getFileAndHash(serviceAccountCAFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	saTokenByte, saTokenHash, err := getFileAndHash(serviceAccountTokenFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caUnchanged := prevCAHash != nil && bytes.Equal(prevCAHash, caHash)
+	saUnchanged := prevSATokenHash != nil && bytes.Equal(prevSATokenHash, saTokenHash)
+
+	if o.SkipTLSVerify {
+		if saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	} else {
+		if caUnchanged && saUnchanged {
+			return caHash, saTokenHash, nil
+		}
+	}
+
+	if prevSATokenHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("CA (%v) or SA token (%v) changed - recreating kubeconfig\n", !caUnchanged, !saUnchanged)
+	}
+
+	// create multus.d directory
+	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
+		return nil, nil, fmt.Errorf("cannot create multus.d directory: %v", err)
+	}
+
+	// create multus cni conf directory
+	if err := os.MkdirAll(o.MultusCNIConfDir, 0755); err != nil {
+		return nil, nil, fmt.Errorf("cannot create multus-cni-conf-dir(%s) directory: %v", o.MultusCNIConfDir, err)
+	}
+
+	// get Kubernetes service protocol/host/port
+	kubeProtocol := os.Getenv("KUBERNETES_SERVICE_PROTOCOL")
+	if kubeProtocol == "" {
+		kubeProtocol = "https"
+	}
+	kubeHost := os.Getenv("KUBERNETES_SERVICE_HOST")
+	kubePort := os.Getenv("KUBERNETES_SERVICE_PORT")
+
+	// check tlsConfig
+	tlsConfig := ""
+	if o.SkipTLSVerify {
+		tlsConfig = "insecure-skip-tls-verify: true"
+	} else {
+		// create tlsConfig by service account CA file
+		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
+		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
+	}
+
+	// create kubeconfig by template and replace it by atomic
+	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
+	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
+	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
+	}
+
+	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
+	if err != nil {
+		return nil, nil, fmt.Errorf("template parse error: %v", err)
+	}
+	templateData := map[string]string{
+		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
+		"KubeServerTLS":           tlsConfig,
+		"KubeServiceAccountToken": string(saTokenByte),
+	}
+
+	// generate kubeconfig from template
+	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
+		return nil, nil, fmt.Errorf("cannot create kubeconfig: %v", err)
+	}
+
+	if err := fp.Sync(); err != nil {
+		os.Remove(fp.Name())
+		return nil, nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
+	}
+	if err := fp.Close(); err != nil {
+		os.Remove(fp.Name())
+		return nil, nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
+	}
+
+	// replace file with tempfile
+	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
+		return nil, nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
+	}
+
+	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
+	return caHash, saTokenHash, nil
+}
+
+const multusConflistTemplate = `{
+    "cniVersion": "{{ .CNIVersion }}",
+    "name": "{{ .MasterPluginNetworkName }}",
+    "plugins": [ {
+        "type": "multus",{{
+            .NestedCapabilities
+        }}{{
+            .NamespaceIsolationConfig
+        }}{{
+            .GlobalNamespacesConfig
+        }}{{
+            .LogToStderrConfig
+        }}{{
+            .LogLevelConfig
+        }}{{
+            .LogFileConfig
+        }}{{
+            .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
+        }}{{
+            .ReadinessIndicatorFileConfig
+        }}
+        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
+        "delegates": [
+            {{ .MasterPluginJSON }}
+        ]
+    }]
+}
+`
+
+const multusConfTemplate = `{
+        "cniVersion": "{{ .CNIVersion }}",
+        "name": "{{ .MasterPluginNetworkName }}",
+        "type": "multus",{{
+            .NestedCapabilities
+        }}{{
+            .NamespaceIsolationConfig
+        }}{{
+            .GlobalNamespacesConfig
+        }}{{
+            .LogToStderrConfig
+        }}{{
+            .LogLevelConfig
+        }}{{
+            .LogFileConfig
+        }}{{
+            .AdditionalBinDirConfig
+        }}{{
+            .MultusCNIConfDirConfig
+        }}{{
+            .ReadinessIndicatorFileConfig
+        }}
+        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
+        "delegates": [
+                {{ .MasterPluginJSON }}
+        ]
+}
+`
+
+func (o *Options) getMasterConfigPath() (string, error) {
+	// Master config file is specified
+	if o.MultusMasterCNIFileName != "" {
+		return filepath.Join(o.MultusAutoconfigDir, o.MultusMasterCNIFileName), nil
+	}
+
+	// Pick the alphabetically first config file from MultusAutoconfigDir
+	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
+	if err != nil {
+		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
+	}
+
+	for _, filename := range files {
+		if !strings.HasPrefix(filepath.Base(filename), "00-multus.conf") {
+			return filename, nil
+		}
+	}
+
+	// No config file found
+	return "", fmt.Errorf("cannot find valid master CNI config in %q", o.MultusAutoconfigDir)
+}
+
+func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, []byte, error) {
+	masterConfigPath, err := o.getMasterConfigPath()
+	if err != nil {
+		return "", nil, err
+	}
+
+	masterConfigBytes, masterConfigFileHash, err := getFileAndHash(masterConfigPath)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if prevMasterConfigFileHash != nil && bytes.Equal(prevMasterConfigFileHash, masterConfigFileHash) {
+		return masterConfigPath, masterConfigFileHash, nil
+	}
+
+	if prevMasterConfigFileHash != nil {
+		// don't log "recreating" on first function execution
+		fmt.Printf("master config changed - recreating multus config\n")
+	}
+
+	masterConfig := map[string]interface{}{}
+	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
+		return "", nil, fmt.Errorf("cannot read master CNI config json: %v", err)
+	}
+
+	// check CNIVersion
+	masterCNIVersionElem, ok := masterConfig["cniVersion"]
+	if !ok {
+		return "", nil, fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
+	}
+
+	if o.ForceCNIVersion {
+		masterConfig["cniVersion"] = o.CNIVersion
+		fmt.Printf("force CNI version to %q\n", o.CNIVersion)
+	} else {
+		masterCNIVersion := masterCNIVersionElem.(string)
+		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
+			return "", nil, fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
+		}
+		o.CNIVersion = masterCNIVersion
+	}
+	cniVersionConfig := o.CNIVersion
+
+	// check OverrideNetworkName (if true, get master plugin name, otherwise 'multus-cni-network'
+	masterPluginNetworkName := "multus-cni-network"
+	if o.OverrideNetworkName {
+		masterPluginNetworkElem, ok := masterConfig["name"]
+		if !ok {
+			return "", nil, fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
+		}
+
+		masterPluginNetworkName = masterPluginNetworkElem.(string)
+		fmt.Printf("master plugin name is overrided to %q\n", masterPluginNetworkName)
+	}
+
+	// check capabilities (from master conf, top and 'plugins')
+	masterCapabilities := map[string]bool{}
+	_, isMasterConfList := masterConfig["plugins"]
+
+	if isMasterConfList {
+		masterPluginsElem, ok := masterConfig["plugins"]
+		if !ok {
+			return "", nil, fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
+		}
+		masterPlugins := masterPluginsElem.([]interface{})
+		for _, v := range masterPlugins {
+			pluginFields := v.(map[string]interface{})
+			capabilitiesElem, ok := pluginFields["capabilities"]
+			if ok {
+				capabilities := capabilitiesElem.(map[string]interface{})
+				for k, v := range capabilities {
+					masterCapabilities[k] = v.(bool)
+				}
+			}
+		}
+		fmt.Printf("master capabilities is get from conflist\n")
+	} else {
+		masterCapabilitiesElem, ok := masterConfig["capabilities"]
+		if ok {
+			for k, v := range masterCapabilitiesElem.(map[string]interface{}) {
+				masterCapabilities[k] = v.(bool)
+			}
+		}
+		fmt.Printf("master capabilities is get from conffile\n")
+	}
+	nestedCapabilitiesConf := ""
+	if len(masterCapabilities) != 0 {
+		capabilitiesByte, err := json.Marshal(masterCapabilities)
+		if err != nil {
+			return "", nil, fmt.Errorf("cannot get capabilities map: %v", err)
+		}
+		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
+	}
+
+	// check NamespaceIsolation
+	namespaceIsolationConfig := ""
+	if o.NamespaceIsolation {
+		namespaceIsolationConfig = "\n        \"namespaceIsolation\": true,"
+	}
+
+	// check GlobalNamespaces
+	globalNamespaceConfig := ""
+	if o.GlobalNamespaces != "" {
+		globalNamespaceConfig = fmt.Sprintf("\n        \"globalNamespaces\": %q,", o.GlobalNamespaces)
+	}
+
+	// check MultusLogToStderr
+	logToStderrConfig := ""
+	if !o.MultusLogToStderr {
+		logToStderrConfig = "\n        \"logToStderr\": false,"
+	}
+
+	// check MultusLogLevel (debug/error/panic/verbose) and reject others
+	logLevelConfig := ""
+	logLevelStr := strings.ToLower(o.MultusLogLevel)
+	switch logLevelStr {
+	case "debug", "error", "panic", "verbose":
+		logLevelConfig = fmt.Sprintf("\n        \"logLevel\": %q,", logLevelStr)
+	case "":
+		// no logLevel config, skipped
+	default:
+		return "", nil, fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
+	}
+
+	// check MultusLogFile
+	logFileConfig := ""
+	if o.MultusLogFile != "" {
+		logFileConfig = fmt.Sprintf("\n        \"logFile\": %q,", o.MultusLogFile)
+	}
+
+	// check AdditionalBinDir
+	additionalBinDirConfig := ""
+	if o.AdditionalBinDir != "" {
+		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
+	}
+
+	// check MultusCNIConfDir
+	multusCNIConfDirConfig := ""
+	if o.MultusCNIConfDir != "" {
+		multusCNIConfDirConfig = fmt.Sprintf("\n        \"cniConf\": %q,", o.MultusCNIConfDir)
+	}
+
+	// check ReadinessIndicatorFile
+	readinessIndicatorFileConfig := ""
+	if o.ReadinessIndicatorFile != "" {
+		readinessIndicatorFileConfig = fmt.Sprintf("\n        \"readinessindicatorfile\": %q,", o.ReadinessIndicatorFile)
+	}
+
+	// fill .MasterPluginJSON
+	masterPluginByte, err := json.Marshal(masterConfig)
+	if err != nil {
+		return "", nil, fmt.Errorf("cannot encode master CNI config: %v", err)
+	}
+
+	// generate multus config
+	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
+	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
+	}
+
+	// use conflist template if cniVersionConfig == "1.0.0"
+	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
+	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
+	if err != nil {
+		return "", nil, fmt.Errorf("template parse error: %v", err)
+	}
+
+	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
+		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
+		if err != nil {
+			return "", nil, fmt.Errorf("template parse error: %v", err)
+		}
+	}
+
+	templateData := map[string]string{
+		"CNIVersion":                   cniVersionConfig,
+		"MasterPluginNetworkName":      masterPluginNetworkName,
+		"NestedCapabilities":           nestedCapabilitiesConf,
+		"NamespaceIsolationConfig":     namespaceIsolationConfig,
+		"GlobalNamespacesConfig":       globalNamespaceConfig,
+		"LogToStderrConfig":            logToStderrConfig,
+		"LogLevelConfig":               logLevelConfig,
+		"LogFileConfig":                logFileConfig,
+		"AdditionalBinDirConfig":       additionalBinDirConfig,
+		"MultusCNIConfDirConfig":       multusCNIConfDirConfig,
+		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
+		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
+		"MasterPluginJSON":             string(masterPluginByte),
+	}
+	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
+		return "", nil, fmt.Errorf("cannot create multus cni config: %v", err)
+	}
+
+	if err := fp.Sync(); err != nil {
+		os.Remove(tempFileName)
+		return "", nil, fmt.Errorf("cannot flush multus cni config: %v", err)
+	}
+	if err := fp.Close(); err != nil {
+		os.Remove(tempFileName)
+		return "", nil, fmt.Errorf("cannot close multus cni config: %v", err)
+	}
+
+	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
+		return "", nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
+	}
+
+	if o.RenameConfFile {
+		//masterConfigPath
+		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
+		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
+			return "", nil, fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
+		}
+		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
+	}
+
+	return masterConfigPath, masterConfigFileHash, nil
+}
+
+func main() {
+	opt := Options{}
+	opt.addFlags()
+	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")
+
+	pflag.Parse()
+	if *helpFlag {
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	err := opt.verifyFileExists()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		return
+	}
+
+	// copy multus binary
+	if !opt.SkipMultusBinaryCopy {
+		// Copy
+		if err = cmdutils.CopyFileAtomic(opt.MultusBinFile, opt.CNIBinDir, "_multus", "multus"); err != nil {
+			fmt.Fprintf(os.Stderr, "failed at multus copy: %v\n", err)
+			return
+		}
+	}
+
+	var masterConfigHash, caHash, saTokenHash []byte
+	var masterConfigFilePath string
+	// copy user specified multus conf to CNI conf directory
+	if opt.MultusConfFile != "auto" {
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
+			return
+		}
+		confFileName := filepath.Base(opt.MultusConfFile)
+		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
+		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
+			fmt.Fprintf(os.Stderr, "failed at copy multus conf file: %v\n", err)
+			return
+		}
+		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
+	} else { // auto generate multus config
+		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
+			return
+		}
+		fmt.Printf("kubeconfig file is created.\n")
+		masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+			return
+		}
+		fmt.Printf("multus config file is created.\n")
+	}
+
+	ctx := signals.SetupSignalHandler()
+
+	if opt.CleanupConfigOnExit {
+		defer cleanupMultusConf(&opt)
+	}
+
+	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
+	if watchChanges {
+		fmt.Printf("Entering watch loop...\n")
+		masterConfigExists := true
+
+	outer:
+		for range time.Tick(1 * time.Second) {
+			select {
+			case <-ctx.Done():
+				// signal received break from loop
+				break outer
+			default:
+				// Check kubeconfig and update if different (i.e. service account updated)
+				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
+					return
+				}
+
+				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
+				_, err = os.Stat(masterConfigFilePath)
+
+				// if masterConfigFilePath is no longer exists
+				if os.IsNotExist(err) {
+					if masterConfigExists {
+						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
+					}
+					masterConfigExists = false
+					continue
+				}
+
+				if !masterConfigExists {
+					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
+					masterConfigExists = true
+				}
+
+				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+					return
+				}
+			}
+		}
+	} else {
+		// wait until signal received
+		<-ctx.Done()
+	}
+}
+
+func cleanupMultusConf(opt *Options) {
+	// try remove multus conf
+	if opt.MultusConfFile == "auto" {
+		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+	} else {
+		confFileName := filepath.Base(opt.MultusConfFile)
+		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
+	}
+
+}

cmd/thin_entrypoint/main_test.go

@@ -0,0 +1,544 @@
+package main
+
+// disable dot-imports only for testing
+//revive:disable:dot-imports
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2" //nolint:golint
+	. "github.com/onsi/gomega"    //nolint:golint
+)
+
+// chrootTestHelper performs chroot syscall, returns func to get back to original root or error if occurred
+func chrootTestHelper(path string) (func() error, error) {
+	root, err := os.Open("/")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := syscall.Chroot(path); err != nil {
+		root.Close()
+		return nil, err
+	}
+
+	return func() error {
+		defer root.Close()
+		if err := root.Chdir(); err != nil {
+			return err
+		}
+		return syscall.Chroot(".")
+	}, nil
+}
+
+func TestThinEntrypoint(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "thin_entrypoint")
+}
+
+var _ = Describe("thin entrypoint testing", func() {
+	It("always pass just example", func() {
+		a := 10
+		Expect(a).To(Equal(10))
+	})
+
+	It("Run verifyFileExists() with expected environment, autoconfig", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
+		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
+		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
+		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
+
+		// CNIConfDir
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// CNIBinDir
+		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
+
+		// MultusBinFile
+		Expect(os.WriteFile(multusBinFile, nil, 0744)).To(Succeed())
+
+		// MultusConfFile
+		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
+
+		err = (&Options{
+			CNIConfDir:     cniConfDir,
+			CNIBinDir:      cniBinDir,
+			MultusBinFile:  multusBinFile,
+			MultusConfFile: multusConfFile,
+		}).verifyFileExists()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run verifyFileExists() with invalid environmentMultusConfFile", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		cniConfDir := fmt.Sprintf("%s/cni_conf_dir", tmpDir)
+		cniBinDir := fmt.Sprintf("%s/cni_bin_dir", tmpDir)
+		multusBinFile := fmt.Sprintf("%s/multus_bin", tmpDir)
+		multusConfFile := fmt.Sprintf("%s/multus_conf", tmpDir)
+
+		// CNIConfDir
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// CNIBinDir
+		Expect(os.Mkdir(cniBinDir, 0755)).To(Succeed())
+
+		// MultusConfFile
+		Expect(os.WriteFile(multusConfFile, nil, 0744)).To(Succeed())
+
+		err = (&Options{
+			CNIConfDir:     cniConfDir,
+			CNIBinDir:      cniBinDir,
+			MultusBinFile:  multusBinFile,
+			MultusConfFile: multusConfFile,
+		}).verifyFileExists()
+		Expect(err).To(HaveOccurred())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), default, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), capabilities, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"capabilities": { "bandwidth": true },
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "capabilities": {"bandwidth":true},
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"capabilities":{"bandwidth":true},"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), with options, conf", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "0.3.1",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		err = os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+			NamespaceIsolation:       true,
+			GlobalNamespaces:         "foobar,barfoo",
+			MultusLogToStderr:        true,
+			MultusLogLevel:           "DEBUG",
+			MultusLogFile:            "/tmp/foobar.log",
+			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
+			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult := `{
+        "cniVersion": "0.3.1",
+        "name": "multus-cni-network",
+        "type": "multus",
+        "namespaceIsolation": true,
+        "globalNamespaces": "foobar,barfoo",
+        "logLevel": "debug",
+        "logFile": "/tmp/foobar.log",
+        "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
+        "readinessindicatorfile": "/var/lib/foobar_indicator",
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+                {"cniVersion":"0.3.1","name":"test1","type":"cnitesttype"}
+        ]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conf", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), default, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conf", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), capabilities, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"capabilities": { "bandwidth": true },
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "capabilities": {"bandwidth":true},
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"capabilities":{"bandwidth":true},"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), with options, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/10-testcni.conflist", multusAutoConfigDir), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+			NamespaceIsolation:       true,
+			GlobalNamespaces:         "foobar,barfoo",
+			MultusLogToStderr:        true,
+			MultusLogLevel:           "DEBUG",
+			MultusLogFile:            "/tmp/foobar.log",
+			AdditionalBinDir:         "/tmp/add_bin_dir",
+			MultusCNIConfDir:         "/tmp/multus/net.d",
+			ReadinessIndicatorFile:   "/var/lib/foobar_indicator",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "namespaceIsolation": true,
+        "globalNamespaces": "foobar,barfoo",
+        "logLevel": "debug",
+        "logFile": "/tmp/foobar.log",
+        "binDir": "/tmp/add_bin_dir",
+        "cniConf": "/tmp/multus/net.d",
+        "readinessindicatorfile": "/var/lib/foobar_indicator",
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createMultusConfig(), with options, conflist", func() {
+		// create directory and files
+		tmpDir, err := os.MkdirTemp("", "multus_thin_entrypoint_tmp")
+		Expect(err).NotTo(HaveOccurred())
+
+		multusAutoConfigDir := fmt.Sprintf("%s/auto_conf", tmpDir)
+		cniConfDir := fmt.Sprintf("%s/cni_conf", tmpDir)
+
+		Expect(os.Mkdir(multusAutoConfigDir, 0755)).To(Succeed())
+		Expect(os.Mkdir(cniConfDir, 0755)).To(Succeed())
+
+		// create master CNI config
+		masterCNIConfigFileName := "10-testcni.conf"
+		masterCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test1",
+			"type": "cnitesttype"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, masterCNIConfigFileName), []byte(masterCNIConfig), 0755)).To(Succeed())
+
+		// create another CNI config
+		anotherCNIConfigFileName := "09-test2cni.conf" // Alphabetically before masterCNIConfigFileName
+		anotherCNIConfig := `
+		{
+			"cniVersion": "1.0.0",
+			"name": "test2",
+			"type": "cnitest2type"
+		}`
+		Expect(os.WriteFile(fmt.Sprintf("%s/%s", multusAutoConfigDir, anotherCNIConfigFileName), []byte(anotherCNIConfig), 0755)).To(Succeed())
+
+		masterConfigPath, masterConfigHash, err := (&Options{
+			MultusAutoconfigDir:      multusAutoConfigDir,
+			MultusMasterCNIFileName:  masterCNIConfigFileName,
+			CNIConfDir:               cniConfDir,
+			MultusKubeConfigFileHost: "/etc/foobar_kubeconfig",
+		}).createMultusConfig(nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(masterConfigPath).NotTo(Equal(""))
+		Expect(masterConfigHash).NotTo(Equal(""))
+
+		expectedResult :=
+			`{
+    "cniVersion": "1.0.0",
+    "name": "multus-cni-network",
+    "plugins": [ {
+        "type": "multus",
+        "logToStderr": false,
+        "kubeconfig": "/etc/foobar_kubeconfig",
+        "delegates": [
+            {"cniVersion":"1.0.0","name":"test1","type":"cnitesttype"}
+        ]
+    }]
+}
+`
+		conf, err := os.ReadFile(fmt.Sprintf("%s/00-multus.conflist", cniConfDir))
+		Expect(string(conf)).To(Equal(expectedResult))
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+	It("Run createKubeConfig()", func() {
+		// create temp dir and files
+		tmpDir := GinkgoT().TempDir()
+
+		cniConfDir := "/cni_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, cniConfDir), 0755)).To(Succeed())
+
+		multusConfDir := "/multus_conf"
+		Expect(os.Mkdir(filepath.Join(tmpDir, multusConfDir), 0755)).To(Succeed())
+
+		// Create service account CA file and token file with dummy data
+		svcAccountPath := filepath.Join(tmpDir, "var/run/secrets/kubernetes.io/serviceaccount")
+		Expect(os.MkdirAll(svcAccountPath, 0755)).ToNot(HaveOccurred())
+		svcAccountCAFile := filepath.Join(tmpDir, serviceAccountCAFile)
+		svcAccountTokenFile := filepath.Join(tmpDir, serviceAccountTokenFile)
+		Expect(os.WriteFile(svcAccountCAFile, []byte("dummy-ca-content"), 0644)).To(Succeed())
+		Expect(os.WriteFile(svcAccountTokenFile, []byte("dummy-token-content"), 0644)).To(Succeed())
+
+		// Set up the Options struct
+		options := &Options{
+			CNIConfDir:       cniConfDir,
+			MultusCNIConfDir: multusConfDir,
+		}
+
+		// Run the createKubeConfig function in a chroot env
+		back, err := chrootTestHelper(tmpDir)
+		Expect(err).ToNot(HaveOccurred())
+		caHash, saTokenHash, err := options.createKubeConfig(nil, nil)
+		Expect(back()).ToNot(HaveOccurred())
+		// back to original root
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(caHash).NotTo(BeNil())
+		Expect(saTokenHash).NotTo(BeNil())
+
+		// Verify the kubeconfig file was created successfully
+		kubeConfigPath := filepath.Join(tmpDir, cniConfDir, "multus.d", "multus.kubeconfig")
+		content, err := os.ReadFile(kubeConfigPath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(content).NotTo(BeEmpty())
+
+		// Cleanup
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
+	})
+
+})

deployments/multus-daemonset-crio.yml

@@ -0,0 +1,227 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-version=0.3.1"
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: run
+          mountPath: /run
+          mountPropagation: HostToContainer
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/multus-daemonset-thick.yml

@@ -0,0 +1,249 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "chrootDir": "/hostroot",
+        "cniVersion": "0.3.1",
+        "logLevel": "verbose",
+        "logToStderr": true,
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "socketDir": "/host/run/multus/"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      hostPID: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
+          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            # multus-daemon expects that cnibin path must be identical between pod and container host.
+            # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
+            # not to any other directory, like '/opt/bin' or '/usr/bin'.
+            - name: cnibin
+              mountPath: /opt/cni/bin
+            - name: host-run
+              mountPath: /host/run
+            - name: host-var-lib-cni-multus
+              mountPath: /var/lib/cni/multus
+            - name: host-var-lib-kubelet
+              mountPath: /var/lib/kubelet
+              mountPropagation: HostToContainer
+            - name: host-run-k8s-cni-cncf-io
+              mountPath: /run/k8s.cni.cncf.io
+            - name: host-run-netns
+              mountPath: /run/netns
+              mountPropagation: HostToContainer
+            - name: multus-daemon-config
+              mountPath: /etc/cni/net.d/multus.d
+              readOnly: true
+            - name: hostroot
+              mountPath: /hostroot
+              mountPropagation: HostToContainer
+          env:
+            - name: MULTUS_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus-shim"
+            - "/host/opt/cni/bin/multus-shim"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: hostroot
+          hostPath:
+            path: /
+        - name: multus-daemon-config
+          configMap:
+            name: multus-daemon-config
+            items:
+            - key: daemon-config.json
+              path: daemon-config.json
+        - name: host-run
+          hostPath:
+            path: /run
+        - name: host-var-lib-cni-multus
+          hostPath:
+            path: /var/lib/cni/multus
+        - name: host-var-lib-kubelet
+          hostPath:
+            path: /var/lib/kubelet
+        - name: host-run-k8s-cni-cncf-io
+          hostPath:
+            path: /run/k8s.cni.cncf.io
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

deployments/multus-daemonset.yml

@@ -0,0 +1,236 @@
+# Note:
+#   This deployment file is designed for 'quickstart' of multus, easy installation to test it,
+#   hence this deployment yaml does not care about following things intentionally.
+#     - various configuration options
+#     - minor deployment scenario
+#     - upgrade/update/uninstall scenario
+#   Multus team understand users deployment scenarios are diverse, hence we do not cover
+#   comprehensive deployment scenario. We expect that it is covered by each platform deployment.
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
+        command: ["/thin_entrypoint"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "--cni-conf-dir=/host/etc/cni/net.d"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
+          command: ["/install_multus"]
+          args:
+            - "--type"
+            - "thin"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

doc/configuration.md

@@ -1,277 +0,0 @@
-## Multus-cni Configuration Reference
-
-Following is the example of multus config file, in `/etc/cni/net.d/`.
-(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
-
-```
-{
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-    "confDir": "/etc/cni/multus/net.d",
-    "cniDir": "/var/lib/cni/multus",
-    "binDir": "/opt/cni/bin",
-    "logFile": "/var/log/multus.log",
-    "logLevel": "debug",
-    "capabilities": {
-        "portMappings": true
-    },    
-    "readinessindicatorfile": "",
-    "namespaceIsolation": false,
-"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
-    "clusterNetwork": "defaultCRD",
-    "defaultNetworks": ["sidecarCRD", "flannel"],
-    "systemNamespaces": ["kube-system", "admin"],
-    "multusNamespace": "kube-system",
-"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
-    "delegates": [{
-        "type": "weave-net",
-        "hairpinMode": true
-    }, {
-        "type": "macvlan",
-        ... (snip)
-    }]
-}
-```
-
-* `name` (string, required): the name of the network
-* `type` (string, required): "multus"
-* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
-* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
-* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
-* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
-* `logFile` (string, optional): file path for log file. multus puts log in given file
-* `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
-* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
-* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf).
-* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready
-
-User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
-
-* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
-* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
-* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
-* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
-* `delegates` ([]map,required): number of delegate details in the Multus
-
-### Network selection flow of clusterNetwork/defaultNetworks
-
-Multus will find network for clusterNetwork/defaultNetworks as following sequences:
-
-1. CRD object for given network name, in 'kube-system' namespace
-1. CNI json config file in `confDir`. Given name should be without extention, like .conf/.conflist. (e.g. "test" for "test.conf")
-1. Directory for CNI json config file. Multus will find alphabetically first file for the network
-1. Multus failed to find network. Multus raise error message
-
-## Miscellaneous config
-
-### Default Network Readiness Indicator
-
-You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
-
-For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
-
-In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
-
-Only one option is necessary to configure this functionality:
-
-* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready.
-
-*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
-
-
-### Logging
-
-You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration.
-
-Multus will always log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet. This method is always enabled.
-
-#### Writing to a Log File
-
-Optionally, you may have Multus log to a file on the filesystem. This file will be written locally on each node where Multus is executed. You may configure this via the `LogFile` option in the CNI configuration. By default this additional logging to a flat file is disabled.
-
-For example in your CNI configuration, you may set:
-
-```
-    "LogFile": "/var/log/multus.log",
-```
-
-#### Logging Level
-
-The default logging level is set as `panic` -- this will log only the most critical errors, and is the least verbose logging level.
-
-The available logging level values, in decreasing order of verbosity are:
-
-* `debug`
-* `verbose`
-* `error`
-* `panic`
-
-You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
-
-```
-    "LogLevel": "debug",
-```
-
-### Namespace Isolation
-
-The functionality provided by the `namespaceIsolation` configuration option enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. In other words, the `NetworkAttachmentDefinitions` are isolated to usage within the namespace in which they're created. 
-
-For example, if a pod is created in the namespace called `development`, Multus will not allow networks to be attached when defined by custom resources created in a different namespace, say in the `default` network.
-
-Consider the situation where you have a system that has users of different privilege levels -- as an example, a platform which has two administrators: a Senior Administrator and a Junior Administrator. The Senior Administrator may have access to all namespaces, and some network configurations as used by Multus are considered to be privileged in that they allow access to some protected resources available on the network. However, the Junior Administrator has access to only a subset of namespaces, and therefore it should be assumed that the Junior Administrator cannot create pods in their limited subset of namespaces. The `namespaceIsolation` feature provides for this isolation, allowing pods created in given namespaces to only access custom resources in the same namespace as the pod.
-
-Namespace Isolation is disabled by default.
-
-#### Configuration example
-
-```
-  "namespaceIsolation": true,
-```
-
-#### Usage example
-
-Let's setup an example where we:
-
-* Create a custom resource in a namespace called `privileged`
-* Create a pod in a namespace called `development`, and have annotations that reference a custom resource in the `privileged` namespace. The creation of this pod should be disallowed by Multus (as we'll have the use of the custom resources limited only to those custom resources created within the same namespace as the pod).
-
-Given the above scenario with a Junior & Senior Administrator. You may assume that the Senior Administrator has access to all namespaces, whereas the Junior Administrator has access only to the `development` namespace.
-
-Firstly, we show that we have a number of namespaces available:
-
-```
-# List the available namespaces
-[user@kube-master ~]$ kubectl get namespaces
-NAME          STATUS   AGE
-default       Active   7h27m
-development   Active   3h
-kube-public   Active   7h27m
-kube-system   Active   7h27m
-privileged    Active   4s
-```
-
-We'll create a `NetworkAttachmentDefinition` in the `privileged` namespace.
-
-```
-# Show the network attachment definition we're creating.
-[user@kube-master ~]$ cat cr.yml
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf
-spec:
-  config: '{
-      "cniVersion": "0.3.0",
-      "type": "macvlan",
-      "master": "eth0",
-      "mode": "bridge",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "192.168.1.0/24",
-        "rangeStart": "192.168.1.200",
-        "rangeEnd": "192.168.1.216",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ],
-        "gateway": "192.168.1.1"
-      }
-    }'
-
-# Create that network attachment definition in the privileged namespace
-[user@kube-master ~]$ kubectl create -f cr.yml -n privileged
-networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
-
-# List the available network attachment definitions in the privileged namespace.
-[user@kube-master ~]$ kubectl get networkattachmentdefinition.k8s.cni.cncf.io -n privileged
-NAME           AGE
-macvlan-conf   11s
-```
-
-Next, we'll create a pod with an annotation that references the privileged namespace. Pay particular attention to the annotation that reads `k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf` -- where it contains a reference to a `namespace/configuration-name` formatted network attachment name. In this case referring to the `macvlan-conf` in the namespace called `privileged`.
-
-```
-# Show the yaml for a pod.
-[user@kube-master ~]$ cat example.pod.yml
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "sleep 2000000000000"]
-    image: dougbtv/centos-network
-
-# Create that pod.
-[user@kube-master ~]$ kubectl create -f example.pod.yml -n development
-pod/samplepod created
-```
-
-You'll note that pod fails to spawn successfully. If you check the Multus logs, you'll see an entry such as:
-
-```
-2018-12-18T21:41:32Z [error] GetPodNetwork: namespace isolation violation: podnamespace: development / target namespace: privileged
-```
-
-This error expresses that the pod resides in the namespace named `development` but refers to a `NetworkAttachmentDefinition` outside of that namespace, in this case, the namespace named `privileged`.
-
-In a positive example, you'd instead create the `NetworkAttachmentDefinition` in the `development` namespace, and you'd have an annotation that either A. does not reference a namespace, or B. refers to the same annotation.
-
-A positive example may be:
-
-```
-# Create the same NetworkAttachmentDefinition as above, however in the development namespace
-[user@kube-master ~]$ kubectl create -f cr.yml -n development
-networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
-
-# Show the yaml for a sample pod which references macvlan-conf without a namspace/ format
-[user@kube-master ~]$ cat positive.example.pod
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "sleep 2000000000000"]
-    image: dougbtv/centos-network
-
-# Create that pod.
-[user@kube-master ~]$ kubectl create -f positive.example.pod -n development
-pod/samplepod created
-
-# We can see that this pod has been launched successfully.
-[user@kube-master ~]$ kubectl get pods -n development
-NAME        READY   STATUS    RESTARTS   AGE
-samplepod   1/1     Running   0          31s
-```
-
-### Specify default cluster network in Pod annotations
-
-Users may also specify the default network for any given pod (via annotation), for cases where there are multiple cluster networks available within a Kubernetes cluster.
-
-Example use cases may include:
-
-1. During a migration from one default network to another (e.g. from Flannel to Calico), it may be practical if both network solutions are able to operate in parallel. Users can then control which network a pod should attach to during the transition period.
-2. Some users may deploy multiple cluster networks for the sake of their security considerations, and may desire to specify the default network for individual pods.
-
-Follow these steps to specify the default network on a pod-by-pod basis:
-
-1. First, you need to define all your cluster networks as network-attachment-definition objects.
-
-2. Next, you can specify the network you want in pods with the `v1.multus-cni.io/default-network` annotation. Pods which do not specify this annotation will keep using the CNI as defined in the Multus config file.
-
-```yaml
-apiVersion: v1
-kind: Pod
-metadata:
-name: pod-example
-annotations:
- v1.multus-cni.io/default-network: calico-conf
-...
-```

doc/development.md

@@ -1,30 +0,0 @@
-## Development Information
-
-## How to build the multus-cni?
-
-```
-git clone https://github.com/intel/multus-cni.git
-cd multus-cni
-./build
-```
-
-## How to run CI tests?
-
-Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
-
-```
-sudo ./test.sh
-```
-
-## Logging Best Practices
-
-Following are multus logging best practices:
-
-* Add `logging.Debugf()` at the begining of functions
-* In case of error handling, use `logging.Errorf()` with given error info
-* `logging.Panicf()` only be used for critical errors (it should NOT normally be used)
-
-
-## CI Introduction
-
-TBD

doc/how-to-use.md

@@ -1,551 +0,0 @@
-## Multus CNI usage guide
-
-### Prerequisites
-
-* Kubelet configured to use CNI
-* Kubernetes version with CRD support (generally )
-
-Your Kubelet(s) must be configured to run with the CNI network plugin. Please see [Kubernetes document for CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni) for more details.
-
-### Install Multus
-
-Generally we recommend two options: Manually place a Multus binary in your `/opt/cni/bin`, or use our [quick-start method](quickstart.md) -- which creates a daemonset that has an opinionated way of how to install & configure Multus CNI (recommended).
-
-*Copy Multus Binary into place*
-
-You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/intel/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
-
-    $ cp multus /opt/cni/bin
-
-*Via Daemonset method*
-
-As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
-
-    $ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
-
-If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
-
-### Set up conf file in /etc/cni/net.d/ (Installed automatically by Daemonset)
-
-**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
-
-You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alphabetically first file in the directory. (`"NOTE1"`, `"NOTE2"` are just comments, you can remove them at your configuration)
-
-Execute following commands at all Kubernetes nodes (i.e. master and minions)
-
-```
-$ mkdir -p /etc/cni/net.d
-$ cat >/etc/cni/net.d/30-multus.conf <<EOF
-{
-  "name": "multus-cni-network",
-  "type": "multus",
-  "readinessindicatorfile": "/var/run/flannel/subnet.env",
-  "delegates": [
-    {
-      "NOTE1": "This is example, wrote your CNI config in delegates",
-      "NOTE2": "If you use flannel, you also need to run flannel daemonset before!",
-      "type": "flannel",
-      "name": "flannel.1",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  ],
-  "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-}
-EOF
-```
-
-For the detail, please take a look into [Configuration Reference](configuration.md)
-
-**NOTE: You can use "clusterNetwork"/"defaultNetworks" instead of "delegates", see []() for the detail**
-
-As above config, you need to set `"kubeconfig"` in the config file for NetworkAttachmentDefinition(CRD).
-
-##### Which network will be used for "Pod IP"?
-
-In case of "delegates", the first delegates network will be used for "Pod IP". Otherwise, "clusterNetwork" will be used for "Pod IP".
-
-#### Create ServiceAccount, ClusterRole and its binding
-
-Create resources for multus to access CRD objects as following command:
-
-```
-# Execute following commands at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
-EOF
-```
-
-#### Set up kubeconfig file
-
-Create kubeconfig at master node as following commands:
-
-```
-# Execute following command at Kubernetes master
-$ mkdir -p /etc/cni/net.d/multus.d
-$ SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
-$ SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
-$ KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
-$ KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
-$ KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
-$ cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
-# Kubeconfig file for Multus CNI plugin.
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
-    certificate-authority-data: ${SERVICEACCOUNT_CA}
-users:
-- name: multus
-  user:
-    token: "${SERVICEACCOUNT_TOKEN}"
-contexts:
-- name: multus-context
-  context:
-    cluster: local
-    user: multus
-current-context: multus-context
-EOF
-```
-
-Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
-**NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
-
-```
-$ scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
-```
-
-### Setup CRDs (daemonset automatically does)
-
-**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
-
-Create CRD definition in Kubernetes as following command at master node:
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  version: v1
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string
-EOF
-```
-
-### Create network attachment definition
-
-The 'NetworkAttachmentDefinition' is used to setup the network attachment, i.e. secondary interface for the pod, There are two ways to configure the 'NetworkAttachmentDefinition' as following:
-
-- NetworkAttachmentDefinition with json CNI config
-- NetworkAttachmentDefinition with CNI config file
-
-#### NetworkAttachmentDefinition with json CNI config:
-
-Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-1
-spec:
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "host-local",
-                "ranges": [
-                    [ {
-                         "subnet": "10.10.0.0/16",
-                         "rangeStart": "10.10.1.20",
-                         "rangeEnd": "10.10.3.50",
-                         "gateway": "10.10.0.254"
-                    } ]
-                ]
-            }
-        }'
-EOF
-```
-
-#### NetworkAttachmentDefinition with CNI config file:
-
-If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir ('/etc/cni/multus/net.d', with same name in the 'name' field of CNI config.
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-2
-EOF
-```
-
-```
-# Execute following commands at all Kubernetes nodes (i.e. master and minions)
-$ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
-{
-  "cniVersion": "0.3.0",
-  "type": "macvlan",
-  "name": "macvlan-conf-2",
-  "master": "eth1",
-  "mode": "bridge",
-  "ipam": {
-      "type": "host-local",
-      "ranges": [
-          [ {
-               "subnet": "11.10.0.0/16",
-               "rangeStart": "11.10.1.20",
-               "rangeEnd": "11.10.3.50"
-          } ]
-      ]
-  }
-}
-```
-
-### Run pod with network annotation
-
-#### Lauch pod with text annotation
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-01
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf-1, macvlan-conf-2
-spec:
-  containers:
-  - name: pod-case-01
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-#### Lauch pod with text annotation for NetworkAttachmentDefinition in different namespace
-
-You can also specify NetworkAttachmentDefinition with its namespace as adding `<namespace>/`
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-3
-  namespace: testns1
-spec:
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "host-local",
-                "ranges": [
-                    [ {
-                         "subnet": "12.10.0.0/16",
-                         "rangeStart": "12.10.1.20",
-                         "rangeEnd": "12.10.3.50"
-                    } ]
-                ]
-            }
-        }'
-EOF
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-02
-  annotations:
-    k8s.v1.cni.cncf.io/networks: testns1/macvlan-conf-3
-spec:
-  containers:
-  - name: pod-case-02
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-#### Lauch pod with text annotation with interface name
-
-You can also specify interface name as adding `@<ifname>`.
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-03
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf-1@macvlan1
-spec:
-  containers:
-  - name: pod-case-03
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-#### Lauch pod with json annotation
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-04
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name" : "macvlan-conf-1" },
-            { "name" : "macvlan-conf-2" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-04
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-#### Lauch pod with json annotation for NetworkAttachmentDefinition in different namespace
-
-You can also specify NetworkAttachmentDefinition with its namespace as adding `"namespace": "<namespace>"`.
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-05
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name" : "macvlan-conf-1",
-              "namespace": "testns1" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-05
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-#### Lauch pod with json annotation with interface
-
-You can also specify interface name as adding `"interface": "<ifname>"`.
-
-```
-# Execute following command at Kubernetes master
-$ cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-06
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name" : "macvlan-conf-1",
-              "interface": "macvlan1" },
-            { "name" : "macvlan-conf-2" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-06
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init
-EOF
-```
-
-### Verifying pod network
-
-Following the example of `ip -d address` output of above pod, "pod-case-06":
-
-```
-# Execute following command at Kubernetes master
-$ kubectl exec -it pod-case-06 -- ip -d address
-1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
-    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
-    inet 127.0.0.1/8 scope host lo
-       valid_lft forever preferred_lft forever
-    inet6 ::1/128 scope host
-       valid_lft forever preferred_lft forever
-3: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
-    link/ether 0a:58:0a:f4:02:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
-    veth numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
-    inet 10.244.2.6/24 scope global eth0
-       valid_lft forever preferred_lft forever
-    inet6 fe80::ac66:45ff:fe7c:3a19/64 scope link
-       valid_lft forever preferred_lft forever
-4: macvlan1@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 4e:6d:7a:4e:14:87 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
-    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
-    inet 10.10.1.22/16 scope global macvlan1
-       valid_lft forever preferred_lft forever
-    inet6 fe80::4c6d:7aff:fe4e:1487/64 scope link
-       valid_lft forever preferred_lft forever
-5: net2@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 6e:e3:71:7f:86:f7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
-    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
-    inet 11.10.1.22/16 scope global net2
-       valid_lft forever preferred_lft forever
-    inet6 fe80::6ce3:71ff:fe7f:86f7/64 scope link
-       valid_lft forever preferred_lft forever
-```
-
-| Interface name | Description |
-| --- | --- |
-| lo | loopback |
-| eth0 | Default network interface (flannel) |
-| macvlan1 | macvlan interface (macvlan-conf-1) |
-| net2 | macvlan interface (macvlan-conf-2) |
-
-## Entrypoint Parameters
-
-Multus CNI, when installed using the daemonset-style installation uses an entrypoint script which copies the Multus binary into place, places CNI configurations. This entrypoint takes a variety of parameters for customization.
-
-Typically, you'd modified the daemonset YAML itself to specify these parameters.
-
-For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
-
-```
-  command: ["/entrypoint.sh"]
-  args:
-  - "--multus-conf-file=auto"
-  - "--namespace-isolation=true"
-  - "--multus-log-level=verbose"
-```
-
-Note that some of the defaults have directories inside the root directory named `/host/`, this is because it is deployed as a container and we have host file system locations mapped into this directory inside the container. If you use other directories, you may have to change the mounted volumes.
-
-### Entrypoint script parameters
-
-Each parameter is shown with the default as the value.
-
-    --cni-conf-dir=/host/etc/cni/net.d
-
-This is the configuration directory where Multus will write its configuration file.
-
-    --cni-bin-dir=/host/opt/cni/bin
-
-This the directory in which the Multus binary will be installed.
-
-    --namespace-isolation=false
-
-Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces.
-
-    --multus-bin-file=/usr/src/multus-cni/bin/multus
-
-This option lets you set which binary executable to copy from the container onto the host (into the directory specified by `--cni-bin-dir`), allowing one to copy an alternate version or build of Multus CNI.
-
-    --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
-
-The `--multus-conf-file` is one of two options; it can be set to a source file to be copied into the location specified by `--cni-conf-dir`. Or, to a value of `auto`, that is: `--multus-conf-file=auto`.
-
-The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
-
-In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
-
-    --multus-autoconfig-dir=/host/etc/cni/net.d
-
-Used only with `--multus-conf-file=auto`. This option allows one to set which directory will be used to generate configuration files.
-
-This can be used if you have your CNI configuration stored in an alternate location, or, you have constraints on race conditions where you'd like to generate your default network configuration first, and then only have Multus write its configuration when it finds that configuration -- allowing only Multus to write the CNI configuration in the `--cni-conf-dir`, therefore notifying the Kubelet that the node is in a ready state.
-
-    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
-
-Used only with `--multus-conf-file=auto`. Allows you to specify an alternate path to the Kubeconfig.
-
-    --multus-log-level=
-    --multus-log-file=
-
-Used only with `--multus-conf-file=auto`. See the documentation for logging for which values are permitted.
-
-Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to speicfy CNI spec version.
-
-    --cni-version=
-
-In some cases, the original CNI configuration that the Multus configuration was generated from (using `--multus-conf-file=auto`) may be used as a sort of semaphor for network readiness -- as this model is used by the Kubelet itself. If you need to disable Multus' availablity, you may wish to clean out the generated configuration file when the source file for autogeneration of the config file is no longer present. You can use this functionality by setting:
-
-    --cleanup-config-on-exit=true
-
-When using CRIO, you may need to restart CRIO to get the Multus configuration file to take -- this is rarely necessary.
-
-    --restart-crio=false
-
-Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
-
-    --rename-conf-file=true
-
-When using `--multus-conf-file=auto` you may also care to specify a `binDir` in the configuration, this can be accomplished using the `--additional-bin-dir` option.
-
-    --additional-bin-dir=/opt/multus/bin

docs/configuration.md

@@ -0,0 +1,382 @@
+# Multus-cni Configuration Reference
+
+## Introduction
+
+Aside from setting options for Multus, one of the goals of configuration is to set the configuration for your *default network*. The default network is also sometimes referred as the "primary CNI plugin", the "primary network", or a "default CNI plugin" and is the CNI plugin that is used to implement [the Kubernetes networking model](https://kubernetes.io/docs/concepts/services-networking/#the-kubernetes-network-model) in your cluster. Common examples include Flannel, Weave, Calico, Cillium, and OVN-Kubernetes, among others.
+
+Here we will refer to this as your default CNI plugin or default network.
+
+## Example configuration
+
+Following is the example of multus config file, in `/etc/cni/net.d/`.
+
+Example configuration using `clusterNetwork` (see also [using delegates](#using-delegates))
+
+```
+{
+    "cniVersion": "0.3.1",
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
+    "logFile": "/var/log/multus.log",
+    "logLevel": "debug",
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    },
+    "capabilities": {
+        "portMappings": true
+    },    
+    "namespaceIsolation": false,
+    "clusterNetwork": "/etc/cni/net.d/99-flannel.conf",
+    "defaultNetworks": ["sidecarCRD", "exampleNetwork"],
+    "systemNamespaces": ["kube-system", "admin"],
+    "multusNamespace": "kube-system",
+    allowTryDeleteOnErr: false
+}
+```
+
+## Index of configuration options
+
+This is a general index of options, however note that you must set either the `clusterNetwork` or `delegates` options, see the following sections after the index for details.
+
+* `name` (string, required): The name of the network
+* `type` (string, required): Must be set to the value of "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* [`logToStderr`](#Logging-via-STDERR) (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
+* [`logFile`](#Writing-to-a-Log-File) (string, optional): file path for log file. multus puts log in given file
+* [`logLevel`](#Logging-Level) (string, optional): logging level (values in decreasing order of verbosity: "debug", "error", "verbose", or "panic")
+* [`logOptions`](#Logging-Options) (object, optional): logging option, More detailed log configuration
+* [`namespaceIsolation`](#Namespace-Isolation) (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* [`globalNamespaces`](#Allow-specific-namespaces-to-be-used-across-namespaces-when-using-namespace-isolation): (string, optional): Used only when `namespaceIsolation` is true, allows specification of comma-delimited list of namespaces which may be referred to outside of namespace isolation.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* [`readinessindicatorfile`](#Default-Network-Readiness-Indicator): The path to a file whose existence denotes that the default network is ready
+message to next when some missing error. Defaults to false.
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` (the default value is `kube-system`)
+* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL 
+
+### Using `clusterNetwork`
+
+Using the `clusterNetwork` option and the `delegates` are **mutually exclusive**. If `clusterNetwork` is set, the `delegates` field is *ignored*. 
+
+You **must** set one or the other.
+
+Therefore:
+
+* Set `clusterNetwork` and if this is set, optionally set the `defaultNetworks`.
+* OR you **must** set `delegates`.
+
+Options:
+
+* `clusterNetwork` (string, required if not using `delegates`): the default CNI plugin to be executed.
+* `defaultNetworks` ([]string, optional): Additional / secondary network attachment that is always attached to each pod. 
+
+The following values are valid for both `clusterNetwork` and `defaultNetworks` and are processed in the following order:
+
+* The name of a `NetworkAttachmentDefinition` custom resource in the namespace specified  by the `multusNamespace` configuration option
+* The `"name"` value in the contents of a CNI JSON configuration file in the CNI configuration directory, 
+  * The given name for `clusterNetwork` should match the value for `name` key in the contents of the CNI JSON file (e.g. `"name": "test"` in `my.conf` when `"clusterNetwork": "test"`)
+* A path to a directory containing CNI json configuration files. The alphabetically first file will be used.
+* Absolute file path for CNI config file
+* If none of the above are found using the value, Multus will raise an error.
+
+If for example you have `defaultNetworks` set as:
+
+```
+"defaultNetworks": ["sidecarNetwork", "exampleNetwork"],
+```
+
+In this example, the values in the expression refer to `NetworkAttachmentDefinition` custom resource names. Therefore, there must be `NetworkAttachmentDefinitions` already created with the names `sidecarNetwork` and `exampleNetwork`.
+
+This means that in addition to the cluster network, each pod would be assigned two additional networks by default, and the pod would present three interfaces, e.g. `eth0`, `net1`, and `net2`, with `net1` and `net2` being set by the above described `NetworkAttachmentDefinitions`. Additional attachments as made by setting `k8s.v1.cni.cncf.io/networks` on pods will be made in addition to those set in the `defaultNetworks` configuration option.
+
+### Using `delegates`
+
+If `clusterNetwork` is not set, you **must** use `delegates`.
+
+* `delegates` ([]map, required if not using `clusterNetwork`). List of CNI configurations to be used as your default CNI plugin(s).
+
+Example configuration using `delegates`:
+
+```
+{
+    "cniVersion": "0.3.1",
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
+    "delegates": [{
+        "type": "weave-net",
+        "hairpinMode": true
+    }, {
+        "type": "macvlan",
+        ... (snip)
+    }]
+}
+```
+
+## Configuration Option Details
+
+### Default Network Readiness Indicator
+
+You may desire that your default network becomes ready before attaching networks with Multus. This is disabled by default and not used unless you set the `readinessindicatorfile` option to a non-blank value.
+
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, therefore, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+
+In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
+
+Only one option is necessary to configure this functionality:
+
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready.
+
+*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
+
+
+### Logging
+
+You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration.
+
+#### Logging via STDERR
+
+By default, Multus will log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet.
+
+Optionally, you may disable this method by setting the `logToStderr` option in your CNI configuration:
+
+```
+    "logToStderr": false,
+```
+
+#### Writing to a Log File
+
+Optionally, you may have Multus log to a file on the filesystem. This file will be written locally on each node where Multus is executed. You may configure this via the `LogFile` option in the CNI configuration. By default this additional logging to a flat file is disabled.
+
+For example in your CNI configuration, you may set:
+
+```
+    "logFile": "/var/log/multus.log",
+```
+
+#### Logging Level
+
+The default logging level is set as `panic` -- this will log only the most critical errors, and is the least verbose logging level.
+
+The available logging level values, in decreasing order of verbosity are:
+
+* `debug`
+* `verbose`
+* `error`
+* `panic`
+
+You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
+
+```
+    "logLevel": "debug",
+```
+
+#### Logging Options
+
+If you want a more detailed configuration of the logging, This includes the following parameters:
+
+* `maxAge` the maximum number of days to retain old log files in their filename
+* `maxSize` the maximum size in megabytes of the log file before it gets rotated
+* `maxBackups` the maximum number of days to retain old log files in their filename
+* `compress` compress determines if the rotated log files should be compressed using gzip
+
+For example in your CNI configuration, you may set:
+
+```
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    }
+```
+
+### Namespace Isolation
+
+The functionality provided by the `namespaceIsolation` configuration option enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. In other words, the `NetworkAttachmentDefinitions` are isolated to usage within the namespace in which they're created. 
+
+**NOTE**: The default namespace is special in this scenario. Even with namespace isolation enabled, any pod, in any namespace is allowed to refer to `NetworkAttachmentDefinitions` in the default namespace. This allows you to create commonly used unprivileged `NetworkAttachmentDefinitions` without having to put them in all namespaces. For example, if you had a `NetworkAttachmentDefinition` named `foo` the default namespace, you may reference it in an annotation with: `default/foo`.
+
+**NOTE**: You can also add additional namespaces which can be referred to globally using the `global-namespaces` option (see next section).
+
+For example, if a pod is created in the namespace called `development`, Multus will not allow networks to be attached when defined by custom resources created in a different namespace, say in the `default` network.
+
+Consider the situation where you have a system that has users of different privilege levels -- as an example, a platform which has two administrators: a Senior Administrator and a Junior Administrator. The Senior Administrator may have access to all namespaces, and some network configurations as used by Multus are considered to be privileged in that they allow access to some protected resources available on the network. However, the Junior Administrator has access to only a subset of namespaces, and therefore it should be assumed that the Junior Administrator cannot create pods in their limited subset of namespaces. The `namespaceIsolation` feature provides for this isolation, allowing pods created in given namespaces to only access custom resources in the same namespace as the pod.
+
+Namespace Isolation is disabled by default.
+
+#### Configuration example
+
+```
+  "namespaceIsolation": true,
+```
+
+#### Usage example
+
+Let's setup an example where we:
+
+* Create a custom resource in a namespace called `privileged`
+* Create a pod in a namespace called `development`, and have annotations that reference a custom resource in the `privileged` namespace. The creation of this pod should be disallowed by Multus (as we'll have the use of the custom resources limited only to those custom resources created within the same namespace as the pod).
+
+Given the above scenario with a Junior & Senior Administrator. You may assume that the Senior Administrator has access to all namespaces, whereas the Junior Administrator has access only to the `development` namespace.
+
+Firstly, we show that we have a number of namespaces available:
+
+```
+# List the available namespaces
+[user@kube-master ~]$ kubectl get namespaces
+NAME          STATUS   AGE
+default       Active   7h27m
+development   Active   3h
+kube-public   Active   7h27m
+kube-system   Active   7h27m
+privileged    Active   4s
+```
+
+We'll create a `NetworkAttachmentDefinition` in the `privileged` namespace.
+
+```
+# Show the network attachment definition we're creating.
+[user@kube-master ~]$ cat cr.yml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'
+
+# Create that network attachment definition in the privileged namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n privileged
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# List the available network attachment definitions in the privileged namespace.
+[user@kube-master ~]$ kubectl get networkattachmentdefinition.k8s.cni.cncf.io -n privileged
+NAME           AGE
+macvlan-conf   11s
+```
+
+Next, we'll create a pod with an annotation that references the privileged namespace. Pay particular attention to the annotation that reads `k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf` -- where it contains a reference to a `namespace/configuration-name` formatted network attachment name. In this case referring to the `macvlan-conf` in the namespace called `privileged`.
+
+```
+# Show the yaml for a pod.
+[user@kube-master ~]$ cat example.pod.yml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f example.pod.yml -n development
+pod/samplepod created
+```
+
+You'll note that pod fails to spawn successfully. If you check the Multus logs, you'll see an entry such as:
+
+```
+2018-12-18T21:41:32Z [error] GetNetworkDelegates: namespace isolation enabled, annotation violates permission, pod is in namespace development but refers to target namespace privileged
+```
+
+This error expresses that the pod resides in the namespace named `development` but refers to a `NetworkAttachmentDefinition` outside of that namespace, in this case, the namespace named `privileged`.
+
+In a positive example, you'd instead create the `NetworkAttachmentDefinition` in the `development` namespace, and you'd have an annotation that either A. does not reference a namespace, or B. refers to the same annotation.
+
+A positive example may be:
+
+```
+# Create the same NetworkAttachmentDefinition as above, however in the development namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n development
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# Show the yaml for a sample pod which references macvlan-conf without a namspace/ format
+[user@kube-master ~]$ cat positive.example.pod
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f positive.example.pod -n development
+pod/samplepod created
+
+# We can see that this pod has been launched successfully.
+[user@kube-master ~]$ kubectl get pods -n development
+NAME        READY   STATUS    RESTARTS   AGE
+samplepod   1/1     Running   0          31s
+```
+
+### Allow specific namespaces to be used across namespaces when using namespace isolation
+
+The `globalNamespaces` configuration option is only used when `namespaceIsolation` is set to true. `globalNamespaces` specifies a comma-delimited list of namespaces which can be referred to from outside of any given namespace in which a pod resides.
+
+```
+  "globalNamespaces": "default,namespace-a,namespace-b",
+```
+
+Note that when using `globalNamespaces` the `default` namespace must be specified in the list if you wish to use that namespace, when `globalNamespaces` is not set, the `default` namespace is implied to be used across namespaces.
+
+### Specify default cluster network in Pod annotations
+
+Users may also specify the default network for any given pod (via annotation), for cases where there are multiple cluster networks available within a Kubernetes cluster.
+
+Example use cases may include:
+
+1. During a migration from one default network to another (e.g. from Flannel to Calico), it may be practical if both network solutions are able to operate in parallel. Users can then control which network a pod should attach to during the transition period.
+2. Some users may deploy multiple cluster networks for the sake of their security considerations, and may desire to specify the default network for individual pods.
+
+Follow these steps to specify the default network on a pod-by-pod basis:
+
+1. First, you need to define all your cluster networks as network-attachment-definition objects.
+
+2. Next, you can specify the network you want in pods with the `v1.multus-cni.io/default-network` annotation. Pods which do not specify this annotation will keep using the CNI as defined in the Multus config file.
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+name: pod-example
+annotations:
+ v1.multus-cni.io/default-network: calico-conf
+...
+```

docs/development.md

@@ -0,0 +1,65 @@
+## Development/Support Information
+
+## Which Kubernetes version is supported in multus?
+
+Currently multus team supports Kubernetes that Kubernetes community maintains.
+See [Version Skew Policy](https://kubernetes.io/releases/version-skew-policy/) for the details.
+
+## How to debug multus-cni thin image?
+
+Latest multus uses [distroless](https://github.com/GoogleContainerTools/distroless) container image for its base,
+hence there is no shell command. If you want to execute shell in multus pod, please use `-debug` image (e.g. ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-debug), which has shell.
+
+## How to utilize multus-cni code as library?
+
+Multus now uses [gopkg.in](http://gopkg.in/) to expose its code as library.
+You can use following command to import our code into your go code.
+
+```
+go get gopkg.in/k8snetworkplumbingwg/multus-cni.v4
+```
+
+## How do I submit an issue?
+
+Use GitHub as normally, you'll be presented with an option to submit a issue or enhancement request.
+
+Issues are considered stale after 90 days. After which, the maintainers reserve the right to close an issue.
+
+Typically, we'll tag the submitter and ask for more information if necessary before closing.
+
+If an issue is closed that you don't feel is sufficiently resolved, please feel free to re-open the issue and provide any necessary information.
+
+## How do I build multus-cni?
+
+You can use the built in `./hack/build-go.sh` script!
+
+```
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
+cd multus-cni
+./hack/build-go.sh
+```
+
+## How do I run CI tests?
+
+Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
+
+```
+sudo ./hack/test-go.sh
+```
+
+## What are the best practices for logging?
+
+The following are the best practices for multus logging:
+
+* Add `logging.Debugf()` at the beginning of functions
+* In case of error handling, use `logging.Errorf()` with given error info
+* `logging.Panicf()` only be used for critical errors (it should NOT normally be used)
+
+
+## Multus release schedule
+
+On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.
+
+## Multi-arch builds
+
+Multus is currently built for a number of architectures, however, our testing and validation is only performed against x86 architectures. Our x86 architecture has end to end testing, however, for other architectures, only supported via best effort community contributions.

docs/how-to-use.md

@@ -0,0 +1,767 @@
+## Multus CNI usage guide
+
+### Prerequisites
+
+* Kubelet configured to use CNI
+* Kubernetes version with CRD support (generally )
+
+Your Kubelet(s) must be configured to run with the CNI network plugin. Please see [Kubernetes document for CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni) for more details.
+
+### Install Multus
+
+Generally we recommend two options: Manually place a Multus binary in your `/opt/cni/bin`, or use our [quick-start method](quickstart.md) -- which creates a daemonset that has an opinionated way of how to install & configure Multus CNI (recommended).
+
+*Copy Multus Binary into place*
+
+You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/k8snetworkplumbingwg/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
+
+    cp multus /opt/cni/bin
+
+*Via Daemonset method*
+
+As a [quickstart](quickstart.md), you may apply these YAML files. Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
+
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml  # thin deployment
+
+or
+
+    kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml # thick (client/server) deployment
+
+If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
+
+### Set up conf file in /etc/cni/net.d/ (Installed automatically by Daemonset)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alphabetically first file in the directory. (`"NOTE1"`, `"NOTE2"` are just comments, you can remove them at your configuration)
+
+Execute following commands at all Kubernetes nodes (i.e. master and minions)
+
+```
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/00-multus.conf <<EOF
+{
+  "name": "multus-cni-network",
+  "type": "multus",
+  "readinessindicatorfile": "/run/flannel/subnet.env",
+  "delegates": [
+    {
+      "NOTE1": "This is example, wrote your CNI config in delegates",
+      "NOTE2": "If you use flannel, you also need to run flannel daemonset before!",
+      "type": "flannel",
+      "name": "flannel.1",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  ],
+  "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+}
+EOF
+```
+
+For the detail, please take a look into [Configuration Reference](configuration.md)
+
+**NOTE: You can use "clusterNetwork"/"defaultNetworks" instead of "delegates", see []() for the detail**
+
+As above config, you need to set `"kubeconfig"` in the config file for NetworkAttachmentDefinition(CRD).
+
+##### Which network will be used for "Pod IP"?
+
+In case of "delegates", the first delegates network will be used for "Pod IP". Otherwise, "clusterNetwork" will be used for "Pod IP".
+
+#### Create ServiceAccount, ClusterRole and its binding
+
+Create resources for multus to access CRD objects as following command:
+
+```
+# Execute following commands at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+EOF
+```
+
+#### Set up kubeconfig file
+
+Create kubeconfig at master node as following commands:
+
+```
+# Execute following command at Kubernetes master
+mkdir -p /etc/cni/net.d/multus.d
+SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
+SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
+KUBERNETES_SERVICE_PROTOCOL=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
+KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
+cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+    certificate-authority-data: ${SERVICEACCOUNT_CA}
+users:
+- name: multus
+  user:
+    token: "${SERVICEACCOUNT_TOKEN}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+EOF
+```
+
+Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
+**NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
+
+```
+scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
+```
+
+### Setup CRDs (daemonset automatically does)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+Create CRD definition in Kubernetes as following command at master node:
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+                 type: string
+EOF
+```
+
+### Create network attachment definition
+
+The 'NetworkAttachmentDefinition' is used to setup the network attachment, i.e. secondary interface for the pod, There are two ways to configure the 'NetworkAttachmentDefinition' as following:
+
+- NetworkAttachmentDefinition with json CNI config
+- NetworkAttachmentDefinition with CNI config file
+
+#### NetworkAttachmentDefinition with json CNI config:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "10.10.0.0/16",
+                         "rangeStart": "10.10.1.20",
+                         "rangeEnd": "10.10.3.50",
+                         "gateway": "10.10.0.254"
+                    } ]
+                ]
+            }
+        }'
+EOF
+```
+
+#### NetworkAttachmentDefinition with CNI config file:
+
+If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir ('/etc/cni/multus/net.d', with same name in the 'name' field of CNI config.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-2
+EOF
+```
+
+```
+# Execute following commands at all Kubernetes nodes (i.e. master and minions)
+cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
+{
+  "cniVersion": "0.3.0",
+  "type": "macvlan",
+  "name": "macvlan-conf-2",
+  "master": "eth1",
+  "mode": "bridge",
+  "ipam": {
+      "type": "host-local",
+      "ranges": [
+          [ {
+               "subnet": "11.10.0.0/16",
+               "rangeStart": "11.10.1.20",
+               "rangeEnd": "11.10.3.50"
+          } ]
+      ]
+  }
+}
+EOF
+```
+
+### Run pod with network annotation
+
+#### Launch pod with text annotation
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-01
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1, macvlan-conf-2
+spec:
+  containers:
+  - name: pod-case-01
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with text annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `<namespace>/`
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-3
+  namespace: testns1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "12.10.0.0/16",
+                         "rangeStart": "12.10.1.20",
+                         "rangeEnd": "12.10.3.50"
+                    } ]
+                ]
+            }
+        }'
+EOF
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-02
+  annotations:
+    k8s.v1.cni.cncf.io/networks: testns1/macvlan-conf-3
+spec:
+  containers:
+  - name: pod-case-02
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with text annotation with interface name
+
+You can also specify interface name as adding `@<ifname>`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-03
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1@macvlan1
+spec:
+  containers:
+  - name: pod-case-03
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-04
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-04
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `"namespace": "<namespace>"`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-05
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "namespace": "testns1" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-05
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation with interface
+
+You can also specify interface name as adding `"interface": "<ifname>"`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-06
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "interface": "macvlan1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-06
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+### Verifying pod network
+
+Following the example of `ip -d address` output of above pod, "pod-case-06":
+
+```
+# Execute following command at Kubernetes master
+kubectl exec -it pod-case-06 -- ip -d address
+
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+    inet6 ::1/128 scope host
+       valid_lft forever preferred_lft forever
+3: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
+    link/ether 0a:58:0a:f4:02:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    veth numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.244.2.6/24 scope global eth0
+       valid_lft forever preferred_lft forever
+    inet6 fe80::ac66:45ff:fe7c:3a19/64 scope link
+       valid_lft forever preferred_lft forever
+4: macvlan1@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 4e:6d:7a:4e:14:87 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.10.1.22/16 scope global macvlan1
+       valid_lft forever preferred_lft forever
+    inet6 fe80::4c6d:7aff:fe4e:1487/64 scope link
+       valid_lft forever preferred_lft forever
+5: net2@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 6e:e3:71:7f:86:f7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 11.10.1.22/16 scope global net2
+       valid_lft forever preferred_lft forever
+    inet6 fe80::6ce3:71ff:fe7f:86f7/64 scope link
+       valid_lft forever preferred_lft forever
+```
+
+| Interface name | Description |
+| --- | --- |
+| lo | loopback |
+| eth0 | Default network interface (flannel) |
+| macvlan1 | macvlan interface (macvlan-conf-1) |
+| net2 | macvlan interface (macvlan-conf-2) |
+
+## Specifying a default route for a specific attachment
+
+Typically, the default route for a pod will route traffic over the `eth0` and therefore over the cluster-wide default network. You may wish to specify that a different network attachment will have the default route.
+
+You can achieve this by using the JSON formatted annotation and specifying a `default-route` key.
+
+*NOTE*: It's important that you consider that this may impact some functionality of getting traffic to route over the cluster-wide default network.
+
+For example, we have a this configuration for macvlan:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.2.0/24",
+        "rangeStart": "192.168.2.200",
+        "rangeEnd": "192.168.2.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.2.1"
+      }
+    }'
+EOF
+```
+
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation.
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{
+      "name": "macvlan-conf",
+      "default-route": ["192.168.2.1"]
+    }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+EOF
+```
+
+This will set `192.168.2.1` as the default route over the `net1` interface, such as:
+
+```
+kubectl exec -it samplepod -- ip route
+
+default via 192.168.2.1 dev net1
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169
+10.244.0.0/16 via 10.244.0.1 dev eth0
+```
+
+## Entrypoint Parameters
+
+Multus CNI, when installed using the daemonset-style installation uses an entrypoint script which copies the Multus binary into place, places CNI configurations. This entrypoint takes a variety of parameters for customization.
+
+Typically, you'd modified the daemonset YAML itself to specify these parameters.
+
+For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
+
+```
+  command: ["/thin_entrypoint"]
+  args:
+  - "--multus-conf-file=auto"
+  - "--namespace-isolation=true"
+  - "--multus-log-level=verbose"
+```
+
+Note that some of the defaults have directories inside the root directory named `/host/`, this is because it is deployed as a container and we have host file system locations mapped into this directory inside the container. If you use other directories, you may have to change the mounted volumes.
+
+### Entrypoint script parameters
+
+Each parameter is shown with the default as the value.
+
+    --cni-conf-dir=/host/etc/cni/net.d
+
+This is the configuration directory where Multus will write its configuration file.
+
+    --cni-bin-dir=/host/opt/cni/bin
+
+This the directory in which the Multus binary will be installed.
+
+    --namespace-isolation=false
+
+Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces. See (the configuration guide for more information)[configuration.md].
+
+    --global-namespaces=default,foo,bar
+
+The `--global-namespaces` works only when `--namespace-isolation=true`. This takes a comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled. See (the configuration guide for more information)[configuration.md].
+
+    --multus-bin-file=/usr/src/multus-cni/bin/multus
+
+This option lets you set which binary executable to copy from the container onto the host (into the directory specified by `--cni-bin-dir`), allowing one to copy an alternate version or build of Multus CNI.
+
+    --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
+
+The `--multus-conf-file` is one of two options; it can be set to a source file to be copied into the location specified by `--cni-conf-dir`. Or, to a value of `auto`, that is: `--multus-conf-file=auto`.
+
+The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
+
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will take the alphabetically first configuration there and wrap that into a Multus configuration.
+
+    --multus-autoconfig-dir=/host/etc/cni/net.d
+
+Used only with `--multus-conf-file=auto`. This option allows one to set which directory will be used to generate configuration files.
+
+This can be used if you have your CNI configuration stored in an alternate location, or, you have constraints on race conditions where you'd like to generate your default network configuration first, and then only have Multus write its configuration when it finds that configuration -- allowing only Multus to write the CNI configuration in the `--cni-conf-dir`, therefore notifying the Kubelet that the node is in a ready state.
+
+    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
+
+Used only with `--multus-conf-file=auto`. Allows you to specify an alternate path to the Kubeconfig.
+
+    --multus-master-cni-file-name=
+
+The `--multus-master-cni-file-name` can be used to select the cni file as the master cni, rather than the first file in cni-conf-dir. For example, `--multus-master-cni-file-name=10-calico.conflist`.
+
+    --multus-log-level=
+    --multus-log-file=
+
+Used only with `--multus-conf-file=auto`. See the [documentation for logging](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#logging) for which values are permitted.
+
+Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to specify CNI spec version.
+
+    --cni-version=
+
+In some cases, the original CNI configuration that the Multus configuration was generated from (using `--multus-conf-file=auto`) may be used as a sort of semaphor for network readiness -- as this model is used by the Kubelet itself. If you need to disable Multus' availability, you may wish to clean out the generated configuration file when the source file for autogeneration of the config file is no longer present. You can use this functionality by setting:
+
+    --cleanup-config-on-exit=true
+
+When specifying `--cleanup-config-on-exit=true` the entrypoint script will delete any generated/copied Multus configuration files when entrypoint script
+exits (upon Pod termination). This allows Multus to be safely removed from the cluster when its no longer needed.
+
+In addition, when both `--cleanup-config-on-exit=true` and `--multus-conf-file=auto` are specified, the entrypoint script will watch for changes of the
+master CNI configuration and kubeconfig. when such change detected, the script will re-genereate Multus configuration. Watch can be skipped by setting:
+
+    --skip-config-watch
+
+Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
+
+    --rename-conf-file=true
+
+When using `--multus-conf-file=auto` you may also care to specify a `binDir` in the configuration, this can be accomplished using the `--additional-bin-dir` option.
+
+    --additional-bin-dir=/opt/multus/bin
+
+Sometimes, you may wish to not have the entrypoint copy the binary file onto the host. Potentially, you have another way to copy in a specific version of Multus, for example. By default, it's always copied, but you may disable the copy with:
+
+    --skip-multus-binary-copy=true
+
+If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
+
+    --readiness-indicator-file=/path/to/file
+
+### Run pod with network annotation and Dynamic Resource Allocation driver
+
+> :warning: Dynamic Resource Allocation (DRA) is [currently an alpha](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/),
+> and is subject to change. Please consider this functionality as a preview. The architecture and usage of DRA in
+> Multus CNI may change in the future as this technology matures.
+>
+> The current DRA integration is based on the DRA API for Kubernetes 1.26 to 1.30. With Kubernetes 1.31, the DRA API
+> will change and multus doesn't integrate with the new API yet.
+
+Dynamic Resource Allocation is alternative mechanism to device plugin which allows to requests pod and container
+resources.
+
+The following sections describe how to use DRA with multus and NVIDIA DRA driver. Other DRA networking driver vendors
+should follow similar concepts to make use of multus DRA support.
+
+#### Prerequisite
+
+1. Kubernetes 1.27
+2. Container Runtime with CDI support enabled
+3. Kubernetes runtime-config=resource.k8s.io/v1alpha2
+4. Kubernetes feature-gates=DynamicResourceAllocation=True,KubeletPodResourcesDynamicResources=true
+
+#### Install DRA driver
+
+The current example uses NVIDIA DRA driver for networking. This DRA driver is not publicly available. An alternative to
+this DRA driver is available at [dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver).
+
+#### Create dynamic resource class with NVIDIA network DRA driver
+
+The `ResourceClass` defines the resource pool of `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClass
+metadata:
+  name: sf-pool-1
+driverName: net.resource.nvidia.com
+EOF
+```
+
+#### Create network attachment definition with resource name
+
+The `k8s.v1.cni.cncf.io/resourceName` should match the `ResourceClass` name defined in the section above.
+In this example it is `sf-pool-1`. Multus query the K8s PodResource API to fetch the `resourceClass` name and also
+query the NetworkAttachmentDefinition `k8s.v1.cni.cncf.io/resourceName`. If both has the same name multus send the
+CDI device name in the DeviceID argument.
+
+##### NetworkAttachmentDefinition for ovn-kubernetes example:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: sf-pool-1
+spec:
+  config: '{
+      "cniVersion": "0.4.0",
+      "dns": {},
+      "ipam": {},
+      "logFile": "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+      "logLevel": "4",
+      "logfile-maxage": 5,
+      "logfile-maxbackups": 5,
+      "logfile-maxsize": 100,
+      "name": "ovn-kubernetes",
+      "type": "ovn-k8s-cni-overlay"
+    }'
+EOF
+```
+
+#### Create DRA Resource Claim
+
+Following command creates `ResourceClaim` `sf` which request resource from  `ResourceClass` `sf-pool-1`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaim
+metadata:
+  namespace: default
+  name: sf
+spec:
+  spec:
+    resourceClassName: sf-pool-1
+EOF
+```
+
+#### Launch pod with DRA Resource Claim
+
+Following command Launch a Pod with primiry network `default` and `ResourceClaim` `sf`.
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: default
+  name: test-sf-claim
+  annotations:
+    v1.multus-cni.io/default-network: default
+spec:
+  restartPolicy: Always
+  containers:
+  - name: with-resource
+    image: docker.io/library/ubuntu:22.04
+    command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+    resources:
+      claims:
+      - name: resource
+  resourceClaims:
+  - name: resource
+    source:
+      resourceClaimName: sf
+```

docs/quickstart.md

@@ -1,6 +1,6 @@
 # Quickstart Guide
 
-This guide is intended as a way to get you off the ground, and using Multus CNI to create Kubernetes pods with multiple interfaces. If you're already using Multus and need more detail, see the [comprehensive usage guide](how-to-use.md). This document a quickstart and a getting started guide in one, intended for your first run-through of Multus CNI.
+This guide is intended as a way to get you off the ground, using Multus CNI to create Kubernetes pods with multiple interfaces. If you're already using Multus and need more detail, see the [comprehensive usage guide](how-to-use.md). This document is a quickstart and a getting started guide in one, intended for your first run-through of Multus CNI.
 
 We'll first install Multus CNI, and then we'll setup some configurations so that you can see how multiple interfaces are created for pods.
 
@@ -15,11 +15,13 @@ Two things we'll refer to a number of times through this document are:
 
 Our installation method requires that you first have installed Kubernetes and have configured a default network -- that is, a CNI plugin that's used for your pod-to-pod connectivity. 
 
+We support Kubernetes versions that Kubernetes community supports. Please see [Supported versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions) in Kubernetes document.
+
 To install Kubernetes, you may decide to use [kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/), or potentially [kubespray](https://github.com/kubernetes-sigs/kubespray).
 
 After installing Kubernetes, you must install a default network CNI plugin. If you're using kubeadm, refer to the "[Installing a pod network add-on](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network)" section in the kubeadm documentation. If it's your first time, we generally recommend using Flannel for the sake of simplicity.
 
-Alternatively, for advanced use cases, for installing Multus and a default network plugin at the same time, you may refer to the [Kubernetes Network Plumbing Group's Reference Deployments](https://github.com/K8sNetworkPlumbingWG/reference-deployment).
+Alternatively, for advanced use cases, for installing Multus and a default network plugin at the same time, you may refer to the [Kubernetes Network Plumbing Group's Reference Deployments](https://github.com/k8snetworkplumbingwg/reference-deployment).
 
 To verify that you default network is ready, you may list your Kubernetes nodes with:
 
@@ -27,35 +29,38 @@ To verify that you default network is ready, you may list your Kubernetes nodes
 kubectl get nodes
 ```
 
-In the case that your default network is ready
+In the case that your default network is ready you will see the `STATUS` column also switch to `Ready` for each node.
+
+```
+NAME                  STATUS   ROLES           AGE    VERSION
+master-0              Ready    master          1h     v1.17.1
+master-1              Ready    master          1h     v1.17.1
+master-2              Ready    master          1h     v1.17.1
+```
 
 ## Installation
 
 Our recommended quickstart method to deploy Multus is to deploy using a Daemonset (a method of running pods on each nodes in your cluster), this spins up pods which install a Multus binary and configure Multus for usage.
 
-Firstly, clone this GitHub repository. 
+We'll apply a YAML file with `kubectl` from this repo, which installs the Multus components.
+
+Recommended installation:
 
 ```
-git clone https://github.com/intel/multus-cni.git && cd multus-cni
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 ```
+See the [thick plugin docs](./thick-plugin.md) for more information about this architecture.
 
-If you're using Kubernetes 1.16+, we'll apply a YAML file with `kubectl` from this repo.
+Alternatively, you may install the thin-plugin with:
 
 ```
-$ cat ./images/multus-daemonset.yml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 ```
 
-Or, for Kubernetes versions < 1.16, use the prior version yaml:
-
-```
-$ cat ./images/multus-daemonset-pre-1.16.yml | kubectl apply -f -
-```
-
-
 ### What the Multus daemonset does
 
 * Starts a Multus daemonset, this runs a pod on each node which places a Multus binary on each node in `/opt/cni/bin`
-* Reads the lexigraphically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
+* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus on each node as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
 * Creates a `/etc/cni/net.d/multus.d` directory on each node with authentication information for Multus to access the Kubernetes API.
 
 
@@ -64,10 +69,10 @@ $ cat ./images/multus-daemonset-pre-1.16.yml | kubectl apply -f -
 Generally, the first step in validating your installation is to ensure that the Multus pods have run without error, you may see an overview of those by looking at:
 
 ```
-$ kubectl get pods --all-namespaces | grep -i multus
+kubectl get pods --all-namespaces | grep -i multus
 ```
 
-You may further validate that it has ran by looking at the `/etc/cni/net.d/` directory and ensure that the alphabetically first
+You may further validate that it has ran by looking at the `/etc/cni/net.d/` directory and ensure that the auto-generated `/etc/cni/net.d/00-multus.conf` exists corresponding to the alphabetically first configuration file.
 
 ## Creating additional interfaces
 
@@ -91,11 +96,11 @@ CNI configurations are JSON, and we have a structure here that has a few things
 2. `type`: This tells CNI which binary to call on disk. Each CNI plugin is a binary that's called. Typically, these binaries are stored in `/opt/cni/bin` on each node, and CNI executes this binary. In this case we've specified the `loopback` binary (which create a loopback-type network interface). If this is your first time installing Multus, you might want to verify that the plugins that are in the "type" field are actually on disk in the `/opt/cni/bin` directory.
 3. `additional`: This field is put here as an example, each CNI plugin can specify whatever configuration parameters they'd like in JSON. These are specific to the binary you're calling in the `type` field.
 
-For an even further example -- take a look at the [bridge CNI plugin README](https://github.com/containernetworking/plugins/tree/master/plugins/main/bridge) which shows additional
+For an even further example -- take a look at the [bridge CNI plugin README](https://github.com/containernetworking/plugins/tree/master/plugins/main/bridge) which shows additional details.
 
 If you'd like more information about CNI configuration, you can read [the entire CNI specification](https://github.com/containernetworking/cni/blob/master/SPEC.md). It might also be useful to look at the [CNI reference plugins](https://github.com/containernetworking/plugins) and see how they're configured.
 
-You do not need to reload or refresh the Kubelets when CNI configurations  change. These are read on each creation & deletion of pods. So if you change a configuration, it'll apply the next time a pod is created. Existing pods may need to be restarted if they need the new configuration.
+You do not need to reload or refresh the Kubelets when CNI configurations change. These are read on each creation & deletion of pods. So if you change a configuration, it'll apply the next time a pod is created. Existing pods may need to be restarted if they need the new configuration.
 
 ### Storing a configuration as a Custom Resource
 
@@ -151,7 +156,7 @@ kubectl describe network-attachment-definitions macvlan-conf
 
 ### Creating a pod that attaches an additional interface
 
-We're going to create a pod. This will look familiar as any pod you might have created before, but, we'll have a special `annotations` field -- in this case we'll have an annotation called `k8s.v1.cni.cncf.io/networks`. This field takes a comma delimited list of the names of your `NetworkAttachmentDefinition`s as we created above. Note in the comand below that we have the annotation of `k8s.v1.cni.cncf.io/networks: macvlan-conf` where `macvlan-conf` is the name we used above when we created our configuration.
+We're going to create a pod. This will look familiar as any pod you might have created before, but, we'll have a special `annotations` field -- in this case we'll have an annotation called `k8s.v1.cni.cncf.io/networks`. This field takes a comma delimited list of the names of your `NetworkAttachmentDefinition`s as we created above. Note in the command below that we have the annotation of `k8s.v1.cni.cncf.io/networks: macvlan-conf` where `macvlan-conf` is the name we used above when we created our configuration.
 
 Let's go ahead and create a pod (that just sleeps for a really long time) with this command:
 
@@ -166,18 +171,18 @@ metadata:
 spec:
   containers:
   - name: samplepod
-    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
-    image: dougbtv/centos-network
+    command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: alpine
 EOF
 ```
 
-You may now inspect the pod and see what interfaces interfaces are attached, like so:
+You may now inspect the pod and see what interfaces are attached, like so:
 
 ```
-$ kubectl exec -it samplepod -- ip a
+kubectl exec -it samplepod -- ip a
 ```
 
-You should note that there's 3 interfaces:
+You should note that there are 3 interfaces:
 
 * `lo` a loopback interface
 * `eth0` our default network
@@ -189,7 +194,7 @@ For additional confirmation, use `kubectl describe pod samplepod` and there will
 
 ```
 Annotations:        k8s.v1.cni.cncf.io/networks: macvlan-conf
-                    k8s.v1.cni.cncf.io/networks-status:
+                    k8s.v1.cni.cncf.io/network-status:
                       [{
                           "name": "cbr0",
                           "ips": [
@@ -225,8 +230,8 @@ metadata:
 spec:
   containers:
   - name: samplepod
-    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
-    image: dougbtv/centos-network
+    command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: alpine
 EOF
 ```
 

docs/thick-plugin.md

@@ -0,0 +1,113 @@
+# Multus Thick plugin
+
+Multus CNI can also be deployed using a thick plugin architecture, which is
+characterized by a client/server architecture.
+
+The client - which will be referred to as "shim" - is a binary executable
+located on the Kubernetes node's file-system that
+[speaks CNI](https://github.com/containernetworking/cni/blob/master/SPEC.md#section-2-execution-protocol):
+the runtime - Kubernetes - passes parameters to the plugin via environment
+variables and configuration - which is passed via stdin.
+The plugin returns a result on stdout on success, or an error on stderr if the
+operation fails. Configuration and results are a JSON encoded string.
+
+Once the shim is invoked by the runtime (Kubernetes) it will contact the
+multus-daemon (server) via a unix domain socket which is bind mounted to the
+host's file-system; the multus-daemon is the one that will do all the
+heavy-pulling: fetch the delegate CNI configuration from the corresponding
+`net-attach-def`, compute the `RuntimeConfig`, and finally, invoke the delegate.
+
+It will then return the result of the operation back to the client.
+
+Please refer to the diagram below for a visual representation of the flow
+described above:
+
+```
+┌─────────┐             ┌───────┐           ┌────────┐             ┌──────────┐
+│         │ cni ADD/DEL │       │ REST POST │        │ cni ADD/DEL │          │
+│ runtime ├────────────►│ shim  │===========│ daemon ├────────────►│ delegate │
+│         │<------------│       │           │        │<------------│          │
+└─────────┘             └───────┘           └────────┘             └──────────┘
+```
+
+## How to use it
+
+### Configure Deployment
+
+If your delegate CNI plugin requires some files which is in container host, please update
+update `deployments/multus-daemonset-thick.yml` to add directory into multus-daemon pod.
+For example, flannel requires `/run/flannel/subnet.env`, so you need to mount this directory
+into the multus-daemon pod.
+
+Required directory/files are different for each CNI plugin, so please refer your CNI plugin.
+
+### Deployment
+
+There is a dedicated multus daemonset specification for users wanting to use
+this thick plugin variant. This reference deployment spec of multus can be
+deployed by following these commands:
+
+```bash
+kubectl apply -f deployments/multus-daemonset-thick.yml
+```
+
+### Command line parameters
+
+The available command line parameters are:
+
+- `config`: Defaults to `"/etc/cni/net.d/multus.d/daemon-config.json"`
+- `version`: Prints the daemon config version and exits
+
+### Server / Daemon configuration
+
+The server configuration is encoded in JSON, and allows the following keys:
+
+- `"chrootDir"`: Specify the directory which points to host root from the pod. See 'Chroot configuration' section for the details.
+- `"socketDir"`: Specify the location where the unix domain socket used
+for client/server communication will be located. This is the location where the
+**Daemon** will read the configuration from. Defaults to `"/run/multus"`.
+- `"metricsPort"`: Metrics port (of multus' metric exporter); by default, no port
+is provided.
+- `"logFile"`: the path to where the daemon logs will be persisted.
+- `"logLevel"`: the logging level for the multus daemon logs.
+- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
+as well. By default, it is disabled.
+
+In addition, you can add any configuration which is in [configuration reference](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#multus-cni-configuration-reference). Server configuration override multus CNI configuration (e.g. `/etc/cni/net.d/00-multus.conf`)
+
+Below you can see an example of the daemon configuration:
+```json
+{
+        "chrootDir": "/hostroot",
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "verbose",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus/",
+        "cniVersion": "0.3.1",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "multusAutoconfigDir": "/host/etc/cni/net.d"
+    }
+```
+
+### Client / Shim configuration
+
+The multus shim configuration is encoded in JSON, and essentially is just a
+regular CNI configuration, usually available in `/etc/cni/net.d/00-multus.conf`.
+
+It allows the following keys:
+
+- `"cniVersion"`: the CNI version for the Multus CNI plugin.
+- `"logFile"`:  the path to where the daemon logs will be persisted.
+- `"logLevel"`: the logging level for the multus daemon logs.
+- `"logToStderr"`: enable this to have the daemon multus logs echoed to stderr
+  as well. By default, it is disabled.
+
+#### Chroot configuration
+
+In thick plugin case, delegate CNI plugin is executed by multus-daemon from Pod, hence if the delegate CNI requires resources in container host, for example unix socket or even file, then CNI plugin is failed to execute because multus-daemon runs in Pod. Multus-daemon supports "chrootDir" option which executes delegate CNI under chroot (to container host).
+
+This configuration is enabled in deployments/multus-daemonset-thick.yml as default.

e2e/README.md

@@ -0,0 +1,26 @@
+## Multus e2e test with kind
+
+### Prerequisite
+
+To run the e2e test, you need the following components:
+
+- curl
+- jinjanator
+- docker
+
+### How to test e2e
+
+```
+$ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
+$ cd multus-cni/e2e
+$ ./get_tools.sh
+$ ./generate_yamls.sh
+$ ./setup_cluster.sh
+$ ./test-simple-macvlan1.sh
+```
+
+### How to teardown cluster
+
+```
+$ ./teardown.sh
+```

e2e/generate_yamls.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ ! -d yamls ]; then
+	mkdir yamls
+fi
+
+# specify CNI version (default: 0.4.0)
+export CNI_VERSION=${CNI_VERSION:-0.4.0}
+
+templates_dir="$(dirname $(readlink -f $0))/templates"
+
+# generate yaml files based on templates/*.j2 to yamls directory
+for i in `ls templates/`; do
+	echo $i
+	j2 -e CNI_VERSION ${templates_dir}/$i -o yamls/${i%.j2}
+done
+unset CNI_VERSION

e2e/get_tools.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -o errexit
+
+if [ ! -d bin ]; then
+	mkdir bin
+fi
+
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.22.0/kind-$(uname)-amd64"
+chmod +x ./bin/kind
+curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+chmod +x ./bin/kubectl
+curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
+chmod +x ./bin/koko
+curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+chmod +x ./bin/jq
+wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm

e2e/setup_cluster.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# define the OCI binary to be used. Acceptable values are `docker`, `podman`.
+# Defaults to `docker`.
+OCI_BIN="${OCI_BIN:-docker}"
+
+# define the deployment spec to use when deploying multus.
+# Acceptable values are `multus-daemonset.yml`. `multus-daemonset-thick.yml`.
+# Defaults to `multus-daemonset-thick.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset-thick.yml}"
+# define the dockerfile to build multus.
+# Acceptable values are `Dockerfile`. `Dockerfile.thick`.
+# Defaults to `Dockerfile.thick`.
+MULTUS_DOCKERFILE="${MULTUS_DOCKERFILE:-Dockerfile.thick}"
+
+kind_network='kind'
+if [ "${MULTUS_DOCKERFILE}" != "none" ]; then
+	$OCI_BIN build -t localhost:5000/multus:e2e -f ../images/${MULTUS_DOCKERFILE} ..
+fi
+
+# deploy cluster with kind
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+    kubeadmConfigPatches:
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          pod-manifest-path: "/etc/kubernetes/manifests/"
+          feature-gates: "DynamicResourceAllocation=true,KubeletPodResourcesDynamicResources=true"
+  - role: worker
+# Required by DRA Integration
+##
+featureGates:
+  DynamicResourceAllocation: true
+runtimeConfig:
+  "api/alpha": "true"
+containerdConfigPatches:
+# Enable CDI as described in
+# https://github.com/container-orchestrated-devices/container-device-interface#containerd-configuration
+- |-
+  [plugins."io.containerd.grpc.v1.cri"]
+      enable_cdi = true
+##
+EOF
+
+# load multus image from container host to kind node
+kind load docker-image localhost:5000/multus:e2e
+
+worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
+worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
+
+kind export kubeconfig
+sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
+sleep 1
+kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
+kubectl create -f yamls/$MULTUS_MANIFEST
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
+kubectl create -f yamls/cni-install.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/simple-static-pod.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web
+  annotations:
+    k8s.v1.cni.cncf.io/networks: "bridge-nad"
+spec: 
+  containers: 
+    - name: web 
+      image: centos:8 
+      command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]

e2e/static-pod-nad.yml

@@ -0,0 +1,15 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: bridge-nad
+spec: 
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "testnet",
+    "type": "bridge",
+    "bridge": "testnet0",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "10.10.0.0/16"
+    }
+}'

e2e/teardown.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+#set -o errexit
+
+export PATH=${PATH}:./bin
+
+# delete cluster kind
+kind delete cluster

e2e/templates/cni-install.yml.j2

@@ -0,0 +1,64 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: cni-install-sh
+  namespace: kube-system
+data:
+  install_cni.sh: |
+    cd /tmp
+    wget https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz
+    cd /host/opt/cni/bin
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.4.0.tgz
+    sleep infinite
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: install-cni-plugins
+  namespace: kube-system
+  labels:
+    name: cni-plugins
+spec:
+  selector:
+    matchLabels:
+      name: cni-plugins
+  template:
+    metadata:
+      labels:
+        name: cni-plugins
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: install-cni-plugins
+        image: alpine
+        command: ["/bin/sh", "/scripts/install_cni.sh"]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: scripts
+          configMap:
+            name: cni-install-sh
+            items:
+            - key: install_cni.sh
+              path: install_cni.sh

e2e/templates/default-route1.yml.j2

@@ -0,0 +1,57 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default-route-config
+spec: 
+  config: '{
+            "cniVersion": "{{ CNI_VERSION }}",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config",
+              "ips": [ "10.1.1.21/24" ] ,
+              "default-route": [ "10.1.1.254" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config", 
+              "ips": [ "10.1.1.22/24" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/templates/dra-integration.yml.j2

@@ -0,0 +1,49 @@
+---
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaimTemplate
+metadata:
+  name: gpu.example.com
+spec:
+  spec:
+    resourceClassName: gpu.example.com
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: dra-net 
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: gpu.example.com
+spec:
+  config: '{
+        "cniVersion": "{{ CNI_VERSION }}",
+        "plugins": [{
+            "name": "mynet",
+            "type": "dummy",
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.1.2.0/24"
+            }
+        }]
+    }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dra-integration
+  labels:
+    app: dra-integration
+  annotations:
+    k8s.v1.cni.cncf.io/networks: default/dra-net
+spec:
+  containers:
+  - name: ctr0
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args: ["export; sleep 9999"]
+    resources:
+      claims:
+      - name: gpu
+  resourceClaims:
+  - name: gpu
+    source:
+      resourceClaimTemplateName: gpu.example.com

e2e/templates/multus-daemonset-thick.yml.j2

@@ -0,0 +1,210 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-daemon-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  daemon-config.json: |
+    {
+        "confDir": "/host/etc/cni/net.d",
+        "logToStderr": true,
+        "logLevel": "debug",
+        "logFile": "/tmp/multus.log",
+        "binDir": "/host/opt/cni/bin",
+        "cniDir": "/var/lib/cni/multus",
+        "socketDir": "/host/run/multus",
+        "cniVersion": "{{ CNI_VERSION }}",
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "forceCNIVersion": true,
+        "multusAutoconfigDir": "/host/etc/cni/net.d"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: host-run
+          mountPath: /host/run
+        - name: host-var-lib-cni-multus
+          mountPath: /var/lib/cni/multus
+        - name: host-run-netns
+          mountPath: /run/netns
+          mountPropagation: HostToContainer
+        - name: multus-daemon-config
+          mountPath: /etc/cni/net.d/multus.d
+          readOnly: true
+        - name: kubelet-pod-resources
+          mountPath: /var/lib/kubelet/pod-resources
+          readOnly: true
+        env:
+        - name: MULTUS_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      initContainers:
+      - name: install-multus-shim
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus-shim"
+          - "/host/opt/cni/bin/multus-shim"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: kubelet-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+        - name: multus-daemon-config
+          configMap:
+            name: multus-daemon-config
+            items:
+            - key: daemon-config.json
+              path: daemon-config.json
+        - name: host-run
+          hostPath:
+            path: /run
+        - name: host-var-lib-cni-multus
+          hostPath:
+            path: /var/lib/cni/multus
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/

e2e/templates/multus-daemonset.yml.j2

@@ -0,0 +1,198 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: ["/thin_entrypoint"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--force-cni-version=true"
+        - "--cni-version={{ CNI_VERSION }}"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command: ["/install_multus"]
+        args:
+          - "--type"
+          - "thin"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/templates/simple-macvlan1.yml.j2

@@ -0,0 +1,63 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan1-config
+spec: 
+  config: '{
+            "cniVersion": "{{ CNI_VERSION }}",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "capabilities": { "ips": true },
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                }, {
+                    "type": "tuning"
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.11/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.12/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker2

e2e/templates/simple-pod.yml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: simple-centos1
+  annotations:
+  labels:
+    app: simple
+spec:
+  containers:
+  - name: simple-centos1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/test-default-route1.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f yamls/default-route1.yml
+kubectl wait --for=condition=ready -l app=default-route1 --timeout=300s pod
+
+echo "check default-route-worker1 interface: net1"
+kubectl exec default-route-worker1 -- ip a show dev net1
+if [ $? -ne 0 ];then
+       exit 1
+fi
+
+echo "check default-route-worker1 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.21" ]; then
+	echo "default-route-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker1 default route"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.1.1.254" ]; then
+	echo "default-route-worker1 default route is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 interface: net1"
+kubectl exec default-route-worker2 -- ip a show dev net1
+if [ $? -ne 0 ];then
+       exit 1
+fi
+
+echo "check default-route-worker2 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.22" ]; then
+	echo "default-route-worker2 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 default route"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.244.1.1" ]; then
+	echo "default-route-worker2 default route is different: ${ipaddr}"
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/default-route1.yml

e2e/test-dra-integration.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# This test is using an example implementation of a DRA driver. This driver is mocking GPU resources. At our test we
+# don't care about what these resources are. We want to ensure that such resource is correctly passed in the Pod using
+# Multus configurations. A couple of notes:
+# - We explitictly pin the revision of the dra-example-driver to the branch `classic-dra` to indicate that the
+#   integration continues to work even when the dra-example-driver is updated. We know that classic-dra is supported
+#   in Kubernetes versions 1.26 to 1.30. Multus supports DRA in the aforementioned Kubernetes versions.
+# - The chart and latest is image is not published somewhere, therefore we have to build locally. This leads to slower
+#   e2e suite runs.
+echo "installing dra-example-driver"
+repo_path="repos/dra-example-driver"
+
+rm -rf $repo_path || true
+git clone --branch classic-dra https://github.com/kubernetes-sigs/dra-example-driver.git ${repo_path}
+${repo_path}/demo/build-driver.sh
+KIND_CLUSTER_NAME=kind ${repo_path}/demo/scripts/load-driver-image-into-kind.sh
+chart_path=${repo_path}/deployments/helm/dra-example-driver/
+overriden_values_path=${chart_path}/overriden_values.yaml
+
+# With the thick plugin, in kind, the primary network on the control plane is not always working as expected. The pods
+# sometimes are not able to communicate with the control plane and the error looks like this:
+# failed to list *v1alpha2.PodSchedulingContext: Get "https://10.96.0.1:443/apis/resource.k8s.io/v1alpha2/podschedulingcontexts?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
+# We override the values here to schedule the controller on the worker nodes where the network is working as expected.
+cat <<EOF >> ${overriden_values_path}
+controller:
+  nodeSelector: null
+  tolerations: null
+EOF
+
+helm install \
+    -n dra-example-driver \
+    --create-namespace \
+    -f ${overriden_values_path} \
+    dra-example-driver \
+    ${chart_path}
+
+echo "installing testing pods"
+kubectl create -f yamls/dra-integration.yml
+kubectl wait --for=condition=ready -l app=dra-integration --timeout=300s pod
+
+echo "check dra-integration pod for DRA injected environment variable"
+
+# We can validate that the resource is correctly injected by checking an environment variable this dra driver is injecting
+# in the Pod.
+# https://github.com/kubernetes-sigs/dra-example-driver/blob/be2b8b1db47b8c757440e955ce5ced88c23bfe86/cmd/dra-example-kubeletplugin/cdi.go#L71C20-L71C44
+env_variable=$(kubectl exec dra-integration -- bash -c "echo \$DRA_RESOURCE_DRIVER_NAME | grep gpu.resource.example.com")
+if [ $? -eq 0 ];then
+	echo "dra-integration pod has DRA injected environment variable"
+else
+	echo "dra-integration pod doesn't have DRA injected environment variable"
+	exit 1
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/dra-integration.yml
+helm uninstall -n dra-example-driver dra-example-driver

e2e/test-simple-macvlan1.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f yamls/simple-macvlan1.yml
+kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
+
+echo "check macvlan1-worker1 interface: net1"
+net=$(kubectl exec macvlan1-worker1 -- ip a show dev net1)
+if [ $? -eq 0 ];then
+	echo "macvlan1-worker1 pod has net1 card"
+else
+	echo "macvlan1-worker1 pod has no net1 card"
+	exit 1
+fi
+
+echo "check macvlan1-worker1 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.11" ]; then
+	echo "macvlan1-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check macvlan1-worker2 interface: net1"
+net2=$(kubectl exec macvlan1-worker2 -- ip a show dev net1)
+if [ $? -eq 0  ];then
+	echo "macvlan1-worker2 pod has net1 card"
+else
+	echo "macvlan1-worker2 pod has no net1 card"
+	exit 1
+fi
+
+echo "check macvlan1-worker2 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.12" ]; then
+	echo "macvlan1-worker2 IP address is different: ${ipaddr}"
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/simple-macvlan1.yml

e2e/test-simple-pod.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f yamls/simple-pod.yml
+kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
+
+echo "cleanup resources"
+kubectl delete -f yamls/simple-pod.yml

e2e/test-static-pod.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o errexit
+
+echo "Creating network attachment definition"
+kubectl create -f static-pod-nad.yml
+
+echo "Creating static pod config file"
+docker cp simple-static-pod.yml kind-worker:/etc/kubernetes/manifests/static-web.yaml
+
+echo "Waiting for static pod to start"
+kubectl wait --for=condition=Ready --namespace=default pod/static-web-kind-worker
+
+echo "Checking the pod annotation for net1 interface"
+kubectl exec static-web-kind-worker --namespace=default -- ip a show dev net1
+
+echo "Deleting static pod"
+docker exec kind-worker /bin/bash -c "rm /etc/kubernetes/manifests/static-web.yaml"
+
+echo "Deleting network attachment definition"
+kubectl delete -f static-pod-nad.yml
+
+echo "Test complete"

examples/README.md

@@ -62,9 +62,9 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
 
 ## Passing down device information
-Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotation in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
 
-In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+In this example (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
 
 ```yaml
 apiVersion: "k8s.cni.cncf.io/v1"
@@ -89,6 +89,6 @@ spec:
   }
 }'
 ```
-The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition.
+The [sriov-pod.yml](./sriov-pod.yml) is an example Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachment definition.
 
->For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

examples/clusterrole.yml

@@ -1,19 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus-crd-overpowered
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update

examples/cni-configuration.conf

@@ -1,13 +0,0 @@
-{
-  "name": "multus-cni-network",
-  "type": "multus",
-  "delegates": [
-    {
-      "type": "flannel",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  ],
-  "kubeconfig": "/etc/kubernetes/kubelet.conf"
-}

examples/crd.yml

@@ -1,21 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  version: v1
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string

examples/flannel-conf.yml

@@ -1,12 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: flannel-conf
-spec: 
-  config: '{
-    "cniVersion": "0.3.0",
-    "type": "flannel",
-    "delegate": {
-      "isDefaultGateway": true
-    }
-  }'

examples/macvlan-conf.yml

@@ -1,21 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf
-spec: 
-  config: '{
-      "cniVersion": "0.3.0",
-      "type": "macvlan",
-      "master": "eth0",
-      "mode": "bridge",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "192.168.1.0/24",
-        "rangeStart": "192.168.1.200",
-        "rangeEnd": "192.168.1.216",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ],
-        "gateway": "192.168.1.1"
-      }
-    }'

examples/macvlan-pod.yml

@@ -0,0 +1,57 @@
+---
+# This net-attach-def defines macvlan-conf with 
+#   + ips capabilities to specify ip in pod annotation and 
+#   + mac capabilities to specify mac address in pod annotation
+# default gateway is defined as well
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec: 
+  config: '{
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "macvlan",
+          "capabilities": { "ips": true },
+          "master": "eth0",
+          "mode": "bridge",
+          "ipam": {
+            "type": "static",
+            "routes": [
+              {
+                "dst": "0.0.0.0/0",
+                "gw": "10.1.1.1"
+              }
+            ] 
+          }
+        }, {
+          "capabilities": { "mac": true },
+          "type": "tuning"
+        }
+      ]
+    }'
+---
+# Define a pod with macvlan-conf, defined above, with ip address and mac, and 
+# "gateway" overrides default gateway to use macvlan-conf's one. 
+# without "gateway" in k8s.v1.cni.cncf.io/networks, default route will be cluster
+# network interface, eth0, even tough macvlan-conf has default gateway config.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan-conf",
+              "ips": [ "10.1.1.101/24" ],
+              "mac": "c2:b0:57:49:47:f1",
+              "gateway": [ "10.1.1.1" ]
+            }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+    ports:
+    - containerPort: 80
+  automountServiceAccountToken: false

examples/multus-ptp-portmap.conf

@@ -1,36 +0,0 @@
-{
-    "name": "multus-cni-network",
-    "type": "multus"
-    "capabilities": {
-        "portMappings": true
-    },
-    "delegates": [
-        {
-            "cniVersion": "0.3.1",
-            "name": "ptp-tuning-conflist",
-            "plugins": [
-                {
-                    "dns": {
-                        "nameservers": [
-                            "172.16.1.1"
-                        ]
-                    },
-                    "ipMasq": true,
-                    "ipam": {
-                        "subnet": "172.16.0.0/24",
-                        "type": "host-local"
-                    },
-                    "mtu": 512,
-                    "type": "ptp"
-                },
-                {
-                    "capabilities": {
-                        "portMappings": true
-                    },
-                    "externalSetMarkChain": "KUBE-MARK-MASQ",
-                    "type": "portmap"
-                }
-            ]
-        }
-    ],
-}

examples/multus-with-flannel.yml

@@ -1,163 +0,0 @@
-# -----------------------------------------------
-# - Example Configuration Deployment
-# -----------------------------------------------
-# - Deploys a .conf file on each node
-# - Configured for Multus + Flannel.
-# - As well as assets for Flannel
-# - Based on https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml
-# -----------------------------------------------
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel
-subjects:
-- kind: ServiceAccount
-  name: flannel
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: flannel
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-multus-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "delegates": [
-        {
-          "type": "flannel",
-          "delegate": {
-            "isDefaultGateway": true
-          }
-        }
-      ],
-      "kubeconfig": "/etc/kubernetes/kubelet.conf"
-    }
-  net-conf.json: |
-    {
-      "Network": "10.244.0.0/16",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-multus-with-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: multus-cfg
-          configMap:
-            name: kube-multus-cfg

examples/net-resource-sample-pod.yaml

@@ -1,21 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: testpod1
-  labels:
-    env: test
-  annotations:
-    k8s.v1.cni.cncf.io/networks: sriov-net-a
-spec:
-  containers:
-  - name: appcntr1
-    image: centos/tools
-    imagePullPolicy: IfNotPresent
-    command: [ "/bin/bash", "-c", "--" ]
-    args: [ "while true; do sleep 300000; done;" ]
-    resources:
-      requests:
-        intel.com/sriov: '1'
-      limits:
-        intel.com/sriov: '1'
-  restartPolicy: "Never"

examples/npwg-demo-1/01_crd.yml

@@ -1,30 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  # group name to use for REST API: /apis/<group>/<version>
-  group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
-  scope: Namespaced
-  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
-    plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
-    singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
-    kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string
-

examples/npwg-demo-1/02_clusterrole.yml

@@ -1,16 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: multus
-rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'

examples/npwg-demo-1/03_namespace1.yml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: testns1

examples/npwg-demo-1/04_macvlan1.yml

@@ -1,72 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-1
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.101/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-2
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.102/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-3
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.103/24" }
-                ]
-            }
-        }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf-4
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.104/24" }
-                ]
-            }
-        }'

examples/npwg-demo-1/05_vlan1.yml

@@ -1,19 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: vlan-conf-1-1
-  namespace: testns1
-spec: 
-  config: '{
-                "cniVersion": "0.3.0",
-                "type": "vlan",
-                "master": "eth1",
-                "vlanid": 1,
-                "ipam": {
-                        "type": "static",
-                        "addresses": [
-                        { "address": "172.16.1.101/24"
-                        } ]
-                }
-        }'

examples/npwg-demo-1/06_flannel2.yml

@@ -1,196 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel2
-subjects:
-- kind: ServiceAccount
-  name: flannel2
-  namespace: kube-system
----
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: flannel2
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-flannel2-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-data:
-  flannel2-conf.json: |
-    {
-      "type": "flannel",
-      "name": "flannel-2",
-      "subnetFile": "/run/flannel/flannel2.env",
-      "dataDir": "/var/lib/cni/flannel2",
-      "delegate": {
-        "bridge": "kbr1"
-      }
-    }
-  net-conf.json: |
-    {
-      "Network": "10.144.0.0/16",
-      "SubnetLen": 24,
-      "SubnetMin": "10.144.0.0",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: flannel-etcd
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - etcd
-    - --advertise-client-urls=http://10.1.1.1:12379
-    - --listen-client-urls=http://0.0.0.0:12379
-    - --listen-peer-urls=http://localhost:12380
-    image: quay.io/coreos/etcd:latest
-    name: etcd
-  hostNetwork: true
-  nodeName: kube-master
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: flannel-etcdctl
-  namespace: kube-system
-spec:
-  template:
-    spec:
-      containers:
-      - name: flannel-etcdctl
-        image: quay.io/coreos/etcd:latest
-        command: ["etcdctl"]
-        args: ["--endpoints=http://10.1.1.1:12379", "set", "/flannel2/network/config", '{ "Network": "10.5.0.0/16", "Backend": {"Type": "vxlan", "VNI": 2}}']
-      hostNetwork: true
-      nodeName: kube-master
-      restartPolicy: Never
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-flannel2-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel2
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel2
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/flannel2-conf.json
-        - /etc/cni/multus/net.d/10-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel2
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --etcd-endpoints=http://10.1.1.1:12379
-        - -iface=eth1
-        - -subnet-file=/run/flannel/flannel2.env
-        - -etcd-prefix=/flannel2/network
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          configMap:
-            name: kube-flannel2-cfg
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: flannel-2

examples/npwg-demo-1/07_ptp_tuning_conflist.yml

@@ -1,30 +0,0 @@
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: ptp-tuning-conflist
-spec: 
-  config: '{
-	"cniVersion": "0.3.1",
-	"name": "ptp-tuning-conflist",
-	"plugins": [{
-			"type": "ptp",
-			"ipMasq": true,
-			"mtu": 512,
-			"ipam": {
-				"type": "host-local",
-				"subnet": "172.16.0.0/24"
-			},
-			"dns": {
-				"nameservers": ["172.16.1.1"]
-			}
-		},
-		{
-			"name": "mytuning",
-			"type": "tuning",
-			"sysctl": {
-				"net.core.somaxconn": "500"
-			}
-		}
-	]
-}'

examples/npwg-demo-1/11_pod_case1.yml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-01
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf-1
-spec:
-  containers:
-  - name: pod-case-01
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/12_pod_case2.yml

@@ -1,18 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-02
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-2" },
-            { "name": "vlan-conf-1-1",
-              "namespace": "testns1",
-              "interface": "vlan1-1" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-02
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/13_pod_case3.yml

@@ -1,17 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-03
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-3" },
-            { "name": "macvlan-conf-4" },
-            { "name": "flannel-2" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-03
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/14_pod_case4.yml

@@ -1,11 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-04
-spec:
-  containers:
-  - name: pod-case-04
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/15_pod_case5.yml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-05
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "ptp-tuning-conflist" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-05
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/sample-pod.yml

@@ -1,14 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
-    image: dougbtv/centos-network
-    ports:
-    - containerPort: 80

examples/sriov-net.yaml

@@ -1,21 +0,0 @@
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: sriov-net-a
-  annotations:
-    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
-spec:
-  config: '{
-  "type": "sriov",
-  "vlan": 1000,
-  "ipam": {
-    "type": "host-local",
-    "subnet": "10.56.217.0/24",
-    "rangeStart": "10.56.217.171",
-    "rangeEnd": "10.56.217.181",
-    "routes": [{
-      "dst": "0.0.0.0/0"
-    }],
-    "gateway": "10.56.217.1"
-  }
-}'

examples/sriov-pod.yml

@@ -0,0 +1,48 @@
+# This net-attach-def defines SR-IOV CNI config
+# Please see https://github.com/intel/sriov-cni and https://github.com/intel/sriov-network-device-plugin
+# for its detail.
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod1
+  labels:
+    env: test
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a
+spec:
+  containers:
+  - name: appcntr1
+    image: centos/tools
+    imagePullPolicy: IfNotPresent
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "while true; do sleep 300000; done;" ]
+    resources:
+      requests:
+        intel.com/sriov: '1'
+      limits:
+        intel.com/sriov: '1'
+  restartPolicy: "Never"
+  automountServiceAccountToken: false

go.mod

@@ -1,42 +1,112 @@
-module github.com/intel/multus-cni
+module gopkg.in/k8snetworkplumbingwg/multus-cni.v4
 
-go 1.12
+go 1.21
 
 require (
-	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/containernetworking/cni v0.7.1
-	github.com/containernetworking/plugins v0.8.2
-	github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 // indirect
-	github.com/gogo/protobuf v0.0.0-20170330071051-c0656edd0d9e // indirect
-	github.com/golang/protobuf v1.3.2 // indirect
-	github.com/google/btree v0.0.0-20160524151835-7d79101e329e // indirect
-	github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 // indirect
-	github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect
-	github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7 // indirect
-	github.com/imdario/mergo v0.0.0-20141206190957-6633656539c1 // indirect
-	github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da // indirect
-	github.com/onsi/ginkgo v1.9.0
-	github.com/onsi/gomega v1.6.0
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/pflag v1.0.1 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.4.0 // indirect
-	github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf
-	github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
-	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 // indirect
-	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
-	golang.org/x/sys v0.0.0-20190825160603-fb81701db80f // indirect
-	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/time v0.0.0-20161028155119-f51c12702a4d // indirect
-	google.golang.org/grpc v1.23.0
-	gopkg.in/inf.v0 v0.9.0 // indirect
-	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect
-	k8s.io/api v0.0.0-20180712090710-2d6f90ab1293
-	k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d
-	k8s.io/client-go v0.0.0-20180718001006-59698c7d9724
-	k8s.io/klog v0.0.0-20181108234604-8139d8cb77af // indirect
-	k8s.io/kubernetes v1.13.0
+	github.com/blang/semver v3.5.1+incompatible
+	github.com/containernetworking/cni v1.2.0-rc1
+	github.com/containernetworking/plugins v1.1.0
+	github.com/fsnotify/fsnotify v1.6.0
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.4
+	github.com/onsi/ginkgo/v2 v2.13.2
+	github.com/onsi/gomega v1.30.0
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
+	golang.org/x/net v0.23.0
+	golang.org/x/sys v0.18.0
+	google.golang.org/grpc v1.58.3
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	k8s.io/api v0.29.0
+	k8s.io/apimachinery v0.29.0
+	k8s.io/client-go v1.5.2
+	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.110.1
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kubelet v0.27.5
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+require (
+	github.com/prometheus/client_golang v1.16.0
+	github.com/spf13/pflag v1.0.5
+)
+
+require (
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.7.0 // indirect
+	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+)
+
+replace (
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	k8s.io/api => k8s.io/api v0.29.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.29.0
+	k8s.io/apiserver => k8s.io/apiserver v0.29.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.0
+	k8s.io/client-go => k8s.io/client-go v0.29.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.0
+	k8s.io/code-generator => k8s.io/code-generator v0.29.0
+	k8s.io/component-base => k8s.io/component-base v0.29.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.29.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.29.0
+	k8s.io/cri-api => k8s.io/cri-api v0.29.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.0
+	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.0
+	k8s.io/kubectl => k8s.io/kubectl v0.29.0
+	k8s.io/kubelet => k8s.io/kubelet v0.29.0
+	k8s.io/kubernetes => k8s.io/kubernetes v1.29.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.0
+	k8s.io/metrics => k8s.io/metrics v0.29.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.29.0
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.0
 )

hack/build-go.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -e
+
+DEST_DIR="bin"
+
+if [ ! -d ${DEST_DIR} ]; then
+	mkdir ${DEST_DIR}
+fi
+
+# Specify correspondingGOARCH from TARGETPLATFORM
+if [ "$TARGETPLATFORM" = "linux/amd64" ]; then
+	export GOARCH=amd64
+elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then
+	export GOARCH=arm64
+elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then
+	export GOARCH=arm
+elif [ "$TARGETPLATFORM" = "linux/ppc64le" ]; then
+	export GOARCH=ppc64le
+elif [ "$TARGETPLATFORM" = "linux/s390x" ]; then
+	export GOARCH=s390x
+fi
+
+# version information
+hasGit=true
+git version > /dev/null 2>&1 || hasGit=false
+GIT_SHA=""
+GIT_TREE_STATE=""
+GIT_TAG=""
+GIT_TAG_LAST=""
+RELEASE_STATUS=""
+if $hasGit; then
+	set +e
+	GIT_SHA=$(git rev-parse --short HEAD)
+	# Tree state is "dirty" if there are uncommitted changes, untracked files are ignored
+	GIT_TREE_STATE=$(test -n "`git status --porcelain --untracked-files=no`" && echo "dirty" || echo "clean")
+	# Empty string if we are not building a tag
+	GIT_TAG=$(git describe --tags --abbrev=0 --exact-match 2>/dev/null || true)
+	# Find most recent tag
+	GIT_TAG_LAST=$(git describe --tags --abbrev=0 2>/dev/null || true)
+	set -e
+fi
+
+# VERSION override mechanism if needed
+VERSION=${VERSION:-}
+if [[ -n "${VERSION}" || -n "${GIT_TAG}" ]]; then
+	RELEASE_STATUS=",released"
+fi
+
+if [ -z "$VERSION" ]; then
+	VERSION=$GIT_TAG_LAST
+fi
+# Add version/commit/date into binary
+DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
+COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.version=${VERSION} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.commit=${COMMIT} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.gitTreeState=${GIT_TREE_STATE} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.releaseStatus=${RELEASE_STATUS} \
+	-X gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/multus.date=${DATE}"
+export CGO_ENABLED=${CGO_ENABLED:-0}
+
+# build with go modules
+export GO111MODULE=on
+
+if [ -n "$MODMODE" ]; then
+	BUILD_ARGS=(-mod "$MODMODE")
+fi
+
+echo "Building multus"
+go build -o ${DEST_DIR}/multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" "$@" ./cmd/multus
+echo "Building multus-daemon"
+go build -o "${DEST_DIR}"/multus-daemon ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-daemon
+echo "Building multus-shim"
+go build -o "${DEST_DIR}"/multus-shim ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/multus-shim
+echo "Building install_multus"
+go build -o "${DEST_DIR}"/install_multus ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/install_multus
+echo "Building thin_entrypoint"
+go build -o "${DEST_DIR}"/thin_entrypoint ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/thin_entrypoint
+echo "Building kubeconfig_generator"
+go build -o "${DEST_DIR}"/kubeconfig_generator ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/kubeconfig_generator
+echo "Building cert-approver"
+go build -o "${DEST_DIR}"/cert-approver ${BUILD_ARGS} -ldflags "${LDFLAGS}" ./cmd/cert-approver

hack/test-go.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+# this if... will be removed when gomodules goes default
+if [ "$GO111MODULE" == "off" ]; then
+	echo "Warning: this will be deprecated in near future so please use go modules!"
+
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
+	REPO_PATH="${ORG_PATH}/multus-cni"
+
+	if [ ! -h gopath/src/${REPO_PATH} ]; then
+		mkdir -p gopath/src/${ORG_PATH}
+		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
+	fi
+
+	export GO15VENDOREXPERIMENT=1
+	export GOBIN=${PWD}/bin
+	export GOPATH=${PWD}/gopath
+	bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
+else
+	# test with go modules
+	bash -c "umask 0; go test -v -race -covermode=atomic -coverprofile=coverage.out ./..."
+fi

images/Dockerfile

@@ -0,0 +1,21 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM --platform=$BUILDPLATFORM golang:1.21 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ARG TARGETPLATFORM
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM gcr.io/distroless/base-debian11:latest
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+COPY --from=build /usr/src/multus-cni/bin/install_multus /
+COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+CMD ["/thin_entrypoint"]

images/Dockerfile.debug

@@ -0,0 +1,21 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM --platform=$BUILDPLATFORM golang:1.21 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ARG TARGETPLATFORM
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM gcr.io/distroless/base-debian11:debug
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+COPY --from=build /usr/src/multus-cni/bin/install_multus /
+COPY --from=build /usr/src/multus-cni/bin/thin_entrypoint /
+COPY --from=build /usr/src/multus-cni/bin/kubeconfig_generator /
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+CMD ["/thin_entrypoint"]

images/Dockerfile.openshift

@@ -0,0 +1,23 @@
+# This dockerfile is specific to building Multus for OpenShift
+# The okd-builder image is locally built from https://raw.githubusercontent.com/okd-project/images/main/okd-builder.Dockerfile
+FROM local/okdbuilder:latest as builder
+
+ADD . /usr/src/multus-cni
+
+WORKDIR /usr/src/multus-cni
+ENV GO111MODULE=off
+RUN ./hack/build-go.sh
+
+FROM quay.io/openshift/origin-base:latest
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
+COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
+COPY --from=builder /usr/src/multus-cni/bin/install_multus /
+COPY --from=builder /usr/src/multus-cni/bin/thin_entrypoint /
+
+LABEL io.k8s.display-name="Multus CNI" \
+      io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
+      io.openshift.tags="openshift" \
+      maintainer="Doug Smith <dosmith@redhat.com>"
+
+ENTRYPOINT ["/thin_entrypoint"]

images/Dockerfile.thick

@@ -0,0 +1,17 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.21 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM debian:stable-slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+COPY --from=build /usr/src/multus-cni/bin/cert-approver /
+WORKDIR /
+
+ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

images/README.md

@@ -5,7 +5,7 @@ This is used for distribution of Multus in a Docker image.
 Typically you'd build this from the root of your Multus clone, as such:
 
 ```
-$ docker build -t dougbtv/multus .
+$ docker build -t dougbtv/multus -f images/Dockerfile .
 ```
 
 ---
@@ -15,7 +15,7 @@ $ docker build -t dougbtv/multus .
 You may wish to deploy Multus as a daemonset, you can do so by starting with the example Daemonset shown here:
 
 ```
-$ kubectl create -f ./images/multus-daemonset.yml
+$ kubectl create -f ./deployments/multus-daemonset.yml
 ```
 
 Note: The likely best practice here is to build your own image given the Dockerfile, and then push it to your preferred registry, and change the `image` fields in the Daemonset YAML to reference that image.
@@ -41,9 +41,7 @@ in lexicographical order in cni-conf-dir).
 ./entrypoint.sh
     -h --help
     --cni-conf-dir=/host/etc/cni/net.d
-    --cni-bin-dir=/host/opt/cni/bin
     --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
-    --multus-bin-file=/usr/src/multus-cni/bin/multus
     --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
 ```
 
@@ -65,4 +63,4 @@ Example docker run command:
 $ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus
 ```
 
-Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).
+Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://docs.projectcalico.org/manifests/calico.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).

images/entrypoint.sh

@@ -1,366 +0,0 @@
-#!/bin/bash
-
-# Always exit on errors.
-set -e
-
-# Run a clean up when we exit if configured to do so.
-trap cleanup TERM
-function cleanup {
-  if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == "true" ]; then
-    CONF=$(cat <<-EOF
-        {Multus configuration intentionally invalidated to prevent pods from being scheduled.}
-EOF
-      )
-      echo $CONF > $CNI_CONF_DIR/00-multus.conf
-      log "Multus configuration intentionally invalidated to prevent pods from being scheduled."
-  fi
-}
-
-
-# Set our known directories.
-CNI_CONF_DIR="/host/etc/cni/net.d"
-CNI_BIN_DIR="/host/opt/cni/bin"
-ADDITIONAL_BIN_DIR=""
-MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
-MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
-MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
-MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
-MULTUS_NAMESPACE_ISOLATION=false
-MULTUS_LOG_LEVEL=""
-MULTUS_LOG_FILE=""
-OVERRIDE_NETWORK_NAME=false
-MULTUS_CLEANUP_CONFIG_ON_EXIT=false
-RESTART_CRIO=false
-CRIO_RESTARTED_ONCE=false
-RENAME_SOURCE_CONFIG_FILE=false
-
-# Give help text for parameters.
-function usage()
-{
-    echo -e "This is an entrypoint script for Multus CNI to overlay its binary and "
-    echo -e "configuration into locations in a filesystem. The configuration & binary file "
-    echo -e "will be copied to the corresponding configuration directory. When "
-    echo -e "'--multus-conf-file=auto' is used, 00-multus.conf will be automatically "
-    echo -e "generated from the CNI configuration file of the master plugin (the first file "
-    echo -e "in lexicographical order in cni-conf-dir)."
-    echo -e ""
-    echo -e "./entrypoint.sh"
-    echo -e "\t-h --help"
-    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
-    echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
-    echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
-    echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
-    echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
-    echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
-    echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
-    echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
-    echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
-    echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
-    echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)"
-    echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)"
-    echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)"
-}
-
-function log()
-{
-    echo "$(date --iso-8601=seconds) ${1}"
-}
-
-function error()
-{
-    log "ERR:  {$1}"
-}
-
-function warn()
-{
-    log "WARN: {$1}"
-}
-
-# Parse parameters given as arguments to this script.
-while [ "$1" != "" ]; do
-    PARAM=`echo $1 | awk -F= '{print $1}'`
-    VALUE=`echo $1 | awk -F= '{print $2}'`
-    case $PARAM in
-        -h | --help)
-            usage
-            exit
-            ;;
-        --cni-version)
-            CNI_VERSION=$VALUE
-            ;;
-        --cni-conf-dir)
-            CNI_CONF_DIR=$VALUE
-            ;;
-        --cni-bin-dir)
-            CNI_BIN_DIR=$VALUE
-            ;;
-        --multus-conf-file)
-            MULTUS_CONF_FILE=$VALUE
-            ;;
-        --multus-bin-file)
-            MULTUS_BIN_FILE=$VALUE
-            ;;
-        --multus-kubeconfig-file-host)
-            MULTUS_KUBECONFIG_FILE_HOST=$VALUE
-            ;;
-        --namespace-isolation)
-            MULTUS_NAMESPACE_ISOLATION=$VALUE
-            ;;
-        --multus-log-level)
-            MULTUS_LOG_LEVEL=$VALUE
-            ;;
-        --multus-log-file)
-            MULTUS_LOG_FILE=$VALUE
-            ;;
-        --multus-autoconfig-dir)
-            MULTUS_AUTOCONF_DIR=$VALUE
-            ;;
-        --override-network-name)
-            OVERRIDE_NETWORK_NAME=$VALUE
-            ;;
-        --cleanup-config-on-exit)
-            MULTUS_CLEANUP_CONFIG_ON_EXIT=$VALUE
-            ;;
-        --restart-crio)
-            RESTART_CRIO=$VALUE
-            ;;
-        --rename-conf-file)
-            RENAME_SOURCE_CONFIG_FILE=$VALUE
-            ;;
-        --additional-bin-dir)
-            ADDITIONAL_BIN_DIR=$VALUE
-            ;;
-        *)
-            warn "unknown parameter \"$PARAM\""
-            ;;
-    esac
-    shift
-done
-
-
-# Create array of known locations
-declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE)
-if [ "$MULTUS_CONF_FILE" != "auto" ]; then
-  arr+=($MULTUS_CONF_FILE)
-fi
-
-
-# Loop through and verify each location each.
-for i in "${arr[@]}"
-do
-  if [ ! -e "$i" ]; then
-    warn "Location $i does not exist"
-    exit 1;
-  fi
-done
-
-# Copy files into place and atomically move into final binary name
-cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR/_multus
-mv -f $CNI_BIN_DIR/_multus $CNI_BIN_DIR/multus 
-if [ "$MULTUS_CONF_FILE" != "auto" ]; then
-  cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
-fi
-
-# Make a multus.d directory (for our kubeconfig)
-
-mkdir -p $CNI_CONF_DIR/multus.d
-MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
-
-# ------------------------------- Generate a "kube-config"
-# Inspired by: https://tinyurl.com/y7r2knme
-SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
-KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
-SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
-SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
-
-
-# Check if we're running as a k8s pod.
-if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
-  # We're running as a k8d pod - expect some variables.
-  if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
-    error "KUBERNETES_SERVICE_HOST not set"; exit 1;
-  fi
-  if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
-    error "KUBERNETES_SERVICE_PORT not set"; exit 1;
-  fi
-
-  if [ "$SKIP_TLS_VERIFY" == "true" ]; then
-    TLS_CFG="insecure-skip-tls-verify: true"
-  elif [ -f "$KUBE_CA_FILE" ]; then
-    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
-  fi
-
-  # Write a kubeconfig file for the CNI plugin.  Do this
-  # to skip TLS verification for now.  We should eventually support
-  # writing more complete kubeconfig files. This is only used
-  # if the provided CNI network config references it.
-  touch $MULTUS_KUBECONFIG
-  chmod ${KUBECONFIG_MODE:-600} $MULTUS_KUBECONFIG
-  cat > $MULTUS_KUBECONFIG <<EOF
-# Kubeconfig file for Multus CNI plugin.
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
-    $TLS_CFG
-users:
-- name: multus
-  user:
-    token: "${SERVICEACCOUNT_TOKEN}"
-contexts:
-- name: multus-context
-  context:
-    cluster: local
-    user: multus
-current-context: multus-context
-EOF
-
-else
-  warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
-fi
-
-# ---------------------- end Generate a "kube-config".
-
-# ------------------------------- Generate "00-multus.conf"
-
-function generateMultusConf {
-if [ "$MULTUS_CONF_FILE" == "auto" ]; then
-  log "Generating Multus configuration file using files in $MULTUS_AUTOCONF_DIR..."
-  found_master=false
-  tries=0
-  while [ $found_master == false ]; do
-    MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
-    if [ "$MASTER_PLUGIN" == "" ]; then
-      if [ $tries -lt 600 ]; then
-        if ! (($tries % 5)); then
-          log "Attemping to find master plugin configuration, attempt $tries"
-        fi
-        let "tries+=1"
-        # See if the Multus configuration file exists, if it does then clean it up.
-        if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ] && [ -f "$CNI_CONF_DIR/00-multus.conf" ]; then
-          # But first, check if it has the invalidated configuration in it (otherwise we keep doing this over and over.)
-          if ! grep -q "invalidated" $CNI_CONF_DIR/00-multus.conf; then
-            cleanup
-          fi
-        fi
-        sleep 1;
-      else
-        error "Multus could not be configured: no master plugin was found."
-        exit 1;
-      fi
-    else
-
-      found_master=true
-
-      ISOLATION_STRING=""
-      if [ "$MULTUS_NAMESPACE_ISOLATION" == true ]; then
-        ISOLATION_STRING="\"namespaceIsolation\": true,"
-      fi
-
-      LOG_LEVEL_STRING=""
-      if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
-        case "$MULTUS_LOG_LEVEL" in
-          debug)
-              ;;
-          error)
-              ;;
-          panic)
-              ;;
-          verbose)
-              ;;
-          *)
-              error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
-              usage
-              exit 1     
-        esac
-        LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
-      fi
-
-      LOG_FILE_STRING=""
-      if [ ! -z "${MULTUS_LOG_FILE// }" ]; then
-        LOG_FILE_STRING="\"logFile\": \"$MULTUS_LOG_FILE\","
-      fi
-
-      CNI_VERSION_STRING=""
-      if [ ! -z "${CNI_VERSION// }" ]; then
-        CNI_VERSION_STRING="\"cniVersion\": \"$CNI_VERSION\","
-      fi
-
-      ADDITIONAL_BIN_DIR_STRING=""
-      if [ ! -z "${ADDITIONAL_BIN_DIR// }" ]; then
-        ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\","
-      fi
-
-      if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
-        MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
-            python -c 'import json,sys;print json.load(sys.stdin)["name"]')"
-      else
-        MASTER_PLUGIN_NET_NAME="multus-cni-network"
-      fi
-
-      MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
-      MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
-      log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
-      CONF=$(cat <<-EOF
-        {
-          $CNI_VERSION_STRING
-          "name": "$MASTER_PLUGIN_NET_NAME",
-          "type": "multus",
-          $ISOLATION_STRING
-          $LOG_LEVEL_STRING
-          $LOG_FILE_STRING
-          $ADDITIONAL_BIN_DIR_STRING
-          "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
-          "delegates": [
-            $MASTER_PLUGIN_JSON
-          ]
-        }
-EOF
-      )
-      echo $CONF > $CNI_CONF_DIR/00-multus.conf
-      log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
-      echo $CONF
-      
-      # If we're not performing the cleanup on exit, we can safely rename the config file.
-      if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
-        mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old
-        log "Original master file moved to ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old"
-      fi
-
-      if [ "$RESTART_CRIO" == true ]; then
-        # Restart CRIO only once.
-        if [ "$CRIO_RESTARTED_ONCE" == false ]; then
-          log "Restarting crio"
-          systemctl restart crio
-          CRIO_RESTARTED_ONCE=true
-        fi
-      fi
-    fi
-  done
-fi
-}
-generateMultusConf
-
-# ---------------------- end Generate "00-multus.conf".
-
-# Enter either sleep loop, or watch loop...
-if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
-  log "Entering watch loop..."
-  while true; do
-    # Check and see if the original master plugin configuration exists...
-    if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then
-      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Performing cleanup..."
-      cleanup
-      generateMultusConf
-      log "Continuing watch loop after configuration regeneration..."
-    fi
-    sleep 1
-  done
-else
-  log "Entering sleep (success)..."
-  sleep infinity
-fi

images/multus-daemonset-crio-pre1.16.yml

@@ -1,244 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: network-attachment-definitions.k8s.cni.cncf.io
-spec:
-  group: k8s.cni.cncf.io
-  version: v1
-  scope: Namespaced
-  names:
-    plural: network-attachment-definitions
-    singular: network-attachment-definition
-    kind: NetworkAttachmentDefinition
-    shortNames:
-    - net-attach-def
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            config:
-                 type: string
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "capabilities": {
-        "portMappings": true
-      },
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "default-cni-network",
-          "plugins": [
-            {
-              "type": "flannel",
-              "name": "flannel.1",
-                "delegate": {
-                  "isDefaultGateway": true,
-                  "hairpinMode": true
-                }
-              },
-              {
-                "type": "portmap",
-                "capabilities": {
-                  "portMappings": true
-                }
-              }
-          ]
-        }
-      ],
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-amd64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--override-network-name=true"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-          capabilities:
-            add: ["SYS_ADMIN"]
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: ppc64le
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: multus
-      containers:
-      - name: kube-multus
-        # crio support requires multus:latest for now. support 3.3 or later.
-        image: nfvpe/multus:latest-ppc64le
-        command: ["/entrypoint.sh"]
-        args:
-        - "--cni-bin-dir=/host/usr/libexec/cni"
-        - "--multus-conf-file=auto"
-        - "--override-network-name=true"
-        - "--restart-crio=true"
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "90Mi"
-          limits:
-            cpu: "100m"
-            memory: "90Mi"
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: cnibin
-          mountPath: /host/usr/libexec/cni
-        - name: multus-cfg
-          mountPath: /tmp/multus-conf
-      volumes:
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cnibin
-          hostPath:
-            path: /usr/libexec/cni
-        - name: multus-cfg
-          configMap:
-            name: multus-cni-config
-            items:
-            - key: cni-conf.json
-              path: 70-multus.conf